Feature/add merkle proof sdk

.gitignore

@@ -7,4 +7,5 @@ config/local.toml
 config/prod.toml
 test-data/user-data*/
 my_permanent_leveldb/
-*level_db*/
+*level_db*/
+*.json

README.md

@@ -35,14 +35,27 @@ output_proof_path="global_proof.json"
 
 cargo run --release --package zk-por-cli --bin zk-por-cli prove --cfg-path ${cfg_dir_path} --output-path ${output_proof_path}
 ```
+
+- get-merkle-proof
+```
+cargo run --release --package zk-por-cli --bin zk-por-cli get-merkle-proof --user-id 409e8a8f5a7e34e4fa13c4a44291e4d5a91bce1a4d90c549765b477fea03fdc5 --output-path merkle_proof.json --cfg-path config
+```
+
 - verify
 ```
 global_root_path="global_proof.json"
 
 # optional. If not provided, will skip verifying the inclusion
-arg_inclusion_proof_path="--inclusion-proof-path inclusion_proof.json"
+inclusion_proof_path="merkle_proof.json"
 
-cargo run --features zk-por-core/verifier --release --package zk-por-cli --bin zk-por-cli verify --global-proof-path ${global_root_path} ${arg_inclusion_proof_path}
+cargo run --features zk-por-core/verifier --release --package zk-por-cli --bin zk-por-cli verify --global-proof-path ${global_root_path} --inclusion-proof-path ${inclusion_proof_path}
+```
+
+## cli
+```
+./target/release/zk-por-cli --help
+./target/release/zk-por-cli prove --cfg-path ${cfg_dir_path} --output-path ${output_proof_path}
+./target/release/zk-por-cli get-merkle-proof --user-id 409e8a8f5a7e34e4fa13c4a44291e4d5a91bce1a4d90c549765b477fea03fdc5 --output-path merkle_proof.json --cfg-path config
 ```
 
 ## code coverage

crates/zk-por-cli/src/lib.rs

@@ -1,3 +1,4 @@
 mod constant;
+pub mod merkle_proof;
 pub mod prover;
 pub mod verifier;

crates/zk-por-cli/src/main.rs

@@ -1,8 +1,8 @@
 use std::{path::PathBuf, str::FromStr};
 
 use clap::{Parser, Subcommand};
-use zk_por_cli::{prover::prove, verifier::verify};
-use zk_por_core::error::PoRError;
+use zk_por_cli::{merkle_proof::get_merkle_proof, prover::prove, verifier::verify};
+use zk_por_core::{config::ProverConfig, error::PoRError};
 
 #[derive(Parser)]
 #[command(version, about, long_about = None)]
@@ -24,8 +24,12 @@ pub enum ZkPorCommitCommands {
         output_path: String, // path to output file
     },
     GetMerkleProof {
+        #[arg(short, long)]
+        cfg_path: String, // path to config file
         #[arg(short, long)]
         user_id: String,
+        #[arg(short, long)]
+        output_path: String, // path to output file
     },
     Verify {
         #[arg(short, long)]
@@ -45,10 +49,11 @@ impl Execute for ZkPorCommitCommands {
                 let output_file = PathBuf::from_str(&output_path).unwrap();
                 prove(prover_cfg, output_file)
             }
-            ZkPorCommitCommands::GetMerkleProof { user_id } => {
-                // TODO: implement this
-                _ = user_id;
-                Ok(())
+            ZkPorCommitCommands::GetMerkleProof { cfg_path, user_id, output_path } => {
+                let cfg = zk_por_core::config::ProverConfig::load(&cfg_path)
+                    .map_err(|e| PoRError::ConfigError(e))?;
+                let prover_cfg: ProverConfig = cfg.try_deserialize().unwrap();
+                get_merkle_proof(user_id.to_string(), prover_cfg, output_path.to_string())
             }
             ZkPorCommitCommands::Verify { global_proof_path, inclusion_proof_path } => {
                 let global_proof_path = PathBuf::from_str(&global_proof_path).unwrap();

crates/zk-por-cli/src/merkle_proof.rs

@@ -0,0 +1,59 @@
+use std::{fs::File, io::Write, str::FromStr};
+
+use serde_json::json;
+use zk_por_core::{
+    config::ProverConfig,
+    database::{DataBase, DbOption},
+    error::PoRError,
+    global::GlobalConfig,
+    merkle_proof::MerkleProof,
+    parser::{AccountParser, FileAccountReader, FileManager, FilesCfg},
+};
+
+use crate::constant::RECURSION_BRANCHOUT_NUM;
+
+pub fn get_merkle_proof(
+    user_id: String,
+    cfg: ProverConfig,
+    output_path: String,
+) -> Result<(), PoRError> {
+    let database = DataBase::new(DbOption {
+        user_map_dir: cfg.db.level_db_user_path.to_string(),
+        gmst_dir: cfg.db.level_db_gmst_path.to_string(),
+    });
+
+    let batch_size = cfg.prover.batch_size as usize;
+    let token_num = cfg.prover.num_of_tokens as usize;
+
+    // the path to dump the final generated proof
+    let file_manager = FileManager {};
+    let account_parser = FileAccountReader::new(
+        FilesCfg {
+            dir: std::path::PathBuf::from_str(&cfg.prover.user_data_path).unwrap(),
+            batch_size: cfg.prover.batch_size,
+            num_of_tokens: cfg.prover.num_of_tokens,
+        },
+        &file_manager,
+    );
+    account_parser.log_state();
+    // let mut account_parser: Box<dyn AccountParser> = Box::new(parser);
+
+    let batch_num = account_parser.total_num_of_users().div_ceil(batch_size);
+
+    let global_cfg = GlobalConfig {
+        num_of_tokens: token_num,
+        num_of_batches: batch_num,
+        batch_size: batch_size,
+        recursion_branchout_num: RECURSION_BRANCHOUT_NUM,
+    };
+
+    let merkle_proof = MerkleProof::new_from_user_id(user_id, &database, &global_cfg)
+        .expect("Unable to generate merkle proof");
+
+    let mut file = File::create(output_path.clone())
+        .expect(format!("fail to create proof file at {:#?}", output_path).as_str());
+    file.write_all(json!(merkle_proof).to_string().as_bytes())
+        .expect("fail to write proof to file");
+
+    Ok(())
+}

crates/zk-por-cli/src/verifier.rs

@@ -4,9 +4,12 @@ use std::{fs::File, path::PathBuf};
 // Assuming Proof is defined in lib.rs and lib.rs is in the same crate
 use super::constant::RECURSION_BRANCHOUT_NUM;
 use zk_por_core::{
+    account::Account,
     circuit_config::{get_recursive_circuit_configs, STANDARD_CONFIG},
     circuit_registry::registry::CircuitRegistry,
     error::PoRError,
+    global::GLOBAL_MST,
+    merkle_proof::MerkleProof,
     Proof,
 };
 
@@ -63,9 +66,20 @@ pub fn verify(
     }
     println!("successfully verify the global proof for round {}", round_num);
 
-    // TODO: verify the inclusion proof
     if let Some(merkle_inclusion_path) = merkle_inclusion_path {
-        _ = merkle_inclusion_path;
+        let merkle_path = File::open(&merkle_inclusion_path).unwrap();
+        let reader = std::io::BufReader::new(merkle_path);
+
+        // Parse the JSON as Proof
+        let (proof, account): (MerkleProof, Account) = from_reader(reader).unwrap();
+
+        let global_mst = GLOBAL_MST.get().unwrap();
+        let mut _g = global_mst.read().expect("unable to get a lock");
+        let gmst_root = _g.inner.last().unwrap();
+
+        let _ =
+            proof.verify_merkle_proof(&account, *gmst_root).expect("Invalid Merkle Proof");
+
         println!("successfully verify the inclusion proof for user for round {}", round_num);
     }
 

crates/zk-por-core/src/account.rs

@@ -3,6 +3,7 @@ use plonky2::{
     plonk::config::Hasher,
 };
 use plonky2_field::types::Field;
+use serde::{Deserialize, Serialize};
 
 use crate::{
     database::{DataBase, UserId},
@@ -11,7 +12,7 @@ use crate::{
 use rand::Rng;
 
 /// A struct representing a users account. It represents their equity and debt as a Vector of goldilocks field elements.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Account {
     pub id: String, // 256 bit hex string
     pub equity: Vec<F>,
@@ -34,6 +35,14 @@ impl Account {
         hash
     }
 
+    pub fn get_empty_account_with_user_id(user_id: String, num_of_tokens: usize) -> Account {
+        Self {
+            id: user_id,
+            equity: vec![F::default(); num_of_tokens],
+            debt: vec![F::default(); num_of_tokens],
+        }
+    }
+
     pub fn get_empty_account(num_of_tokens: usize) -> Account {
         Self {
             id: "0".repeat(64),
@@ -70,12 +79,9 @@ pub fn persist_account_id_to_gmst_pos(
         .iter()
         .enumerate()
         .map(|(i, acct)| {
-            let hex_decode = hex::decode(&acct.id).unwrap();
-            assert_eq!(hex_decode.len(), 32);
-            let mut array = [0u8; 32];
-            array.copy_from_slice(&hex_decode);
-
-            (UserId(array), (i + start_idx) as u32)
+            let user_id = UserId::from_hex_string(acct.id.to_string()).unwrap();
+            // tracing::debug!("persist account {:?} with index: {:?}", acct.id, i + start_idx);
+            (user_id, (i + start_idx) as u32)
         })
         .collect::<Vec<(UserId, u32)>>();
     db.add_batch_users(user_batch);
@@ -105,6 +111,10 @@ pub fn gen_accounts_with_random_data(num_accounts: usize, num_assets: usize) ->
 }
 
 pub fn gen_empty_accounts(batch_size: usize, num_assets: usize) -> Vec<Account> {
-    let accounts = vec![Account::get_empty_account(num_assets); batch_size];
+    let accounts =
+        vec![
+            Account::get_empty_account_with_user_id(UserId::rand().to_string(), num_assets);
+            batch_size
+        ];
     accounts
 }

crates/zk-por-core/src/config.rs

@@ -40,6 +40,22 @@ pub struct ConfigDb {
     pub level_db_gmst_path: String,
 }
 
+impl ConfigDb {
+    pub fn load(dir: &str) -> Result<Config, ConfigError> {
+        let env = std::env::var("ENV").unwrap_or("default".into());
+        Config::builder()
+            // .add_source(File::with_name(&format!("{}/default", dir)))
+            .add_source(File::with_name(&format!("{}/{}", dir, env)).required(false))
+            .add_source(File::with_name(&format!("{}/local", dir)).required(false))
+            .add_source(config::Environment::with_prefix("ZKPOR"))
+            .build()
+    }
+    pub fn try_new() -> Result<Self, ConfigError> {
+        let config = Self::load("config")?;
+        config.try_deserialize()
+    }
+}
+
 #[derive(Debug, Clone, Deserialize)]
 pub struct ProverConfig {
     pub log: ConfigLog,

crates/zk-por-core/src/database.rs

@@ -1,10 +1,11 @@
 use std::str::FromStr;
 
+use hex::ToHex;
 use plonky2::{hash::hash_types::HashOut, plonk::config::GenericHashOut};
 use rand::Rng;
 use zk_por_db::LevelDb;
 
-use crate::types::F;
+use crate::{error::PoRError, types::F};
 
 #[derive(Debug, Clone, Copy)]
 pub struct UserId(pub [u8; 32]);
@@ -16,6 +17,29 @@ impl UserId {
         rng.fill(&mut bytes);
         Self(bytes)
     }
+
+    pub fn to_string(&self) -> String {
+        self.0.encode_hex()
+    }
+
+    pub fn from_hex_string(hex_str: String) -> Result<Self, PoRError> {
+        if hex_str.len() != 64 {
+            tracing::error!("User Id: {:?} is not a valid id, length is not 256 bits", hex_str);
+            return Err(PoRError::InvalidParameter(hex_str));
+        }
+
+        let decode_res = hex::decode(hex_str.clone());
+
+        if decode_res.is_err() {
+            tracing::error!("User Id: {:?} is not a valid id", hex_str);
+            return Err(PoRError::InvalidParameter(hex_str));
+        }
+
+        let mut arr = [0u8; 32];
+        arr.copy_from_slice(&decode_res.unwrap());
+
+        Ok(UserId { 0: arr })
+    }
 }
 
 impl db_key::Key for UserId {

crates/zk-por-core/src/error.rs

@@ -6,6 +6,9 @@ pub enum PoRError {
     #[error("Proof is not valid")]
     InvalidProof,
 
+    #[error("Merkle proof is not valid")]
+    InvalidMerkleProof,
+
     #[error("config error: {0}")]
     ConfigError(#[from] ConfigError),
 
@@ -20,4 +23,7 @@ pub enum PoRError {
 
     #[error("The verification circuit digest does not match the prover. ")]
     CircuitDigestMismatch,
+
+    #[error("User is not valid")]
+    InvalidUser,
 }