From e0cc7769605d112a95486663f014ce92853bee7c Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Fri, 16 Aug 2024 15:42:25 +0800 Subject: [PATCH 01/24] feature:add-merkle-verification-logic --- crates/zk-por-core/src/database.rs | 22 +++ crates/zk-por-core/src/merkle_proof.rs | 195 +++++++++++++++++++++++-- 2 files changed, 203 insertions(+), 14 deletions(-) diff --git a/crates/zk-por-core/src/database.rs b/crates/zk-por-core/src/database.rs index d0fdfe4..d8cc4c7 100644 --- a/crates/zk-por-core/src/database.rs +++ b/crates/zk-por-core/src/database.rs @@ -1,5 +1,6 @@ use std::str::FromStr; +use hex::ToHex; use plonky2::{hash::hash_types::HashOut, plonk::config::GenericHashOut}; use rand::Rng; use zk_por_db::LevelDb; @@ -16,6 +17,27 @@ impl UserId { rng.fill(&mut bytes); Self(bytes) } + + pub fn to_string(&self) -> String { + self.0.encode_hex() + } + + pub fn from_hex_string(hex_str: String) -> Self { + if hex_str.len() != 64 { + tracing::error!("User Id: {:?} is not a valid id, length is not 256 bits", hex_str); + } + let mut arr = [0u8; 32]; + for i in 0..32 { + let byte_str = &hex_str[2 * i..2 * i + 2]; + let byte = u8::from_str_radix(byte_str, 16); + if byte.is_err() { + tracing::error!("User Id: {:?} is not a valid id, not in proper hex form", hex_str); + } + arr[i] = byte.unwrap(); + } + + UserId { 0: arr } + } } impl db_key::Key for UserId { diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index cbb5561..aa2fa1d 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -1,11 +1,31 @@ +use itertools::Itertools; +use plonky2::{ + hash::{hash_types::HashOut, poseidon::PoseidonHash}, + plonk::config::Hasher, +}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use crate::global::GlobalMst; +use crate::{ + account::Account, + database::{DataBase, UserId}, + global::GlobalMst, + merkle_sum_prover::utils::{hash_2_subhashes, hash_inputs}, + types::{D, F}, +}; + +/// We use this wrapper struct for the left and right indexes of our recursive siblings. This is needed so a user knows the position of +/// their own hash when hashing. +#[derive(Debug, Clone, PartialEq)] +pub struct RecursiveIndex { + left_indexes: Vec, + right_indexes: Vec, +} -#[derive(Debug, Clone)] +/// Indexes for a given users merkle proof of inclusion siblings in the Global Merkle Sum Tree +#[derive(Debug, Clone, PartialEq)] pub struct MerkleProofIndex { pub sum_tree_siblings: Vec, - pub recursive_tree_siblings: Vec>, + pub recursive_tree_siblings: Vec, } impl MerkleProofIndex { @@ -49,7 +69,7 @@ pub fn get_mst_siblings_index(global_index: usize, global_mst: &GlobalMst) -> Ve pub fn get_recursive_siblings_index( global_index: usize, global_mst: &GlobalMst, -) -> Vec> { +) -> Vec { // Make sure our global index is within the number of leaves assert!(global_index < global_mst.get_num_of_leaves()); @@ -71,29 +91,45 @@ pub fn get_recursive_siblings_index( .ceil() as usize; for i in 0..layers { - let mut layer = Vec::new(); + let mut left_layer = Vec::new(); + let mut right_layer = Vec::new(); if i == 0 { for j in 0..global_mst.cfg.recursion_branchout_num { - if j != recursive_offset { + if j < recursive_offset { let index = first_mst_root_idx + (global_mst.cfg.recursion_branchout_num * recursive_idx) + j; - layer.push(index); + left_layer.push(index); + } + + if j > recursive_offset { + let index = first_mst_root_idx + + (global_mst.cfg.recursion_branchout_num * recursive_idx) + + j; + right_layer.push(index); } } } else { for j in 0..global_mst.cfg.recursion_branchout_num { - if j != recursive_offset { + if j < recursive_offset { + let index = global_mst.get_recursive_global_index( + i, + recursive_idx * global_mst.cfg.recursion_branchout_num + j, + ); + left_layer.push(index); + } + + if j > recursive_offset { let index = global_mst.get_recursive_global_index( i, recursive_idx * global_mst.cfg.recursion_branchout_num + j, ); - layer.push(index); + right_layer.push(index); } } } - siblings.push(layer); + siblings.push(RecursiveIndex { left_indexes: left_layer, right_indexes: right_layer }); recursive_offset = recursive_idx % global_mst.cfg.recursion_branchout_num; recursive_idx = recursive_idx / global_mst.cfg.recursion_branchout_num; @@ -102,11 +138,122 @@ pub fn get_recursive_siblings_index( siblings } +/// Hashes for a given users merkle proof of inclusion siblings in the Global Merkle Sum Tree +#[derive(Debug, Clone)] +pub struct MerkleProof { + pub sum_tree_siblings: Vec>, + pub recursive_tree_siblings: Vec, +} + +/// We use this wrapper struct for the left and right hashes of our recursive siblings. This is needed so a user knows the position of +/// their own hash when hashing. +#[derive(Debug, Clone, PartialEq)] +pub struct RecursiveHashes { + left_hashes: Vec>, + right_hashes: Vec>, +} + +impl RecursiveHashes { + pub fn new_from_index(indexes: &RecursiveIndex, db: &DataBase) -> Self { + let left_hashes = indexes + .left_indexes + .iter() + .map(|y| db.get_gmst_node_hash(*y as i32).unwrap()) + .collect_vec(); + let right_hashes = indexes + .right_indexes + .iter() + .map(|y| db.get_gmst_node_hash(*y as i32).unwrap()) + .collect_vec(); + RecursiveHashes { left_hashes, right_hashes } + } + + pub fn get_calculated_hash(self, own_hash: HashOut) -> HashOut { + let mut hash_inputs = self.left_hashes; + hash_inputs.push(own_hash); + hash_inputs.extend(self.right_hashes); + + let inputs: Vec = hash_inputs.iter().map(|x| x.elements).flatten().collect(); + + PoseidonHash::hash_no_pad(inputs.as_slice()) + } +} + +impl MerkleProof { + pub fn new_from_user_id(user_id: UserId, db: &DataBase, global_mst: &GlobalMst) -> MerkleProof { + let user_index = db.get_user_index(user_id); + if user_index.is_none() { + tracing::error!("User with id: {:?} does not exist", user_id.to_string()); + } + + let indexes = + MerkleProofIndex::new_from_user_index(user_index.unwrap() as usize, global_mst); + let merkle_proof = get_merkle_proof_hashes_from_indexes(&indexes, db); + merkle_proof + } + + pub fn verify_merkle_proof( + &self, + account: &Account, + db: DataBase, + gmst_root: HashOut, + ) -> Result { + let account_hash = account.get_hash(); + let user_index_res = db.get_user_index(UserId::from_hex_string(account.id.clone())); + if user_index_res.is_none() { + tracing::error!("User with id: {:?} does not exist", account.id.to_string()); + } + + let mut user_index = user_index_res.unwrap(); + + let calculated_mst_hash = self.sum_tree_siblings.iter().fold(account_hash, |acc, x| { + if user_index % 2 == 0 { + user_index /= 2; + hash_2_subhashes::(x, &acc) + } else { + user_index /= 2; + hash_2_subhashes::(&acc, x) + } + }); + + let calculated_hash = self + .recursive_tree_siblings + .iter() + .fold(calculated_mst_hash, |acc, x| x.clone().get_calculated_hash(acc)); + + if calculated_hash == gmst_root { + Ok(account.clone()) + } else { + Err("Merkle Proof is not verified".to_string()) + } + } +} + +/// Given the indexes for the MST siblings, get the hashes from the database for the merkle proof of inclusion. +pub fn get_merkle_proof_hashes_from_indexes( + indexes: &MerkleProofIndex, + db: &DataBase, +) -> MerkleProof { + let mst_hashes: Vec> = indexes + .sum_tree_siblings + .iter() + .map(|x| db.get_gmst_node_hash(*x as i32).unwrap()) + .collect(); + + let recursive_hashes: Vec = indexes + .recursive_tree_siblings + .iter() + .map(|x| RecursiveHashes::new_from_index(x, db)) + .collect(); + + MerkleProof { sum_tree_siblings: mst_hashes, recursive_tree_siblings: recursive_hashes } +} + #[cfg(test)] pub mod test { use crate::{ global::{GlobalConfig, GlobalMst}, - merkle_proof::get_recursive_siblings_index, + merkle_proof::{get_recursive_siblings_index, RecursiveIndex}, }; use super::get_mst_siblings_index; @@ -162,7 +309,14 @@ pub mod test { let global_index = 0; let siblings = get_recursive_siblings_index(global_index, &gmst); - assert_eq!(siblings, vec![vec![91, 92, 93], vec![107, 108, 109]]); + + assert_eq!( + siblings, + vec![ + RecursiveIndex { left_indexes: vec![], right_indexes: vec![91, 92, 93] }, + RecursiveIndex { left_indexes: vec![], right_indexes: vec![107, 108, 109] } + ] + ); let gmst = GlobalMst::new(GlobalConfig { num_of_tokens: 100, @@ -174,7 +328,14 @@ pub mod test { let global_index = 163; let siblings = get_recursive_siblings_index(global_index, &gmst); - assert_eq!(siblings, vec![vec![441, 442, 443], vec![456, 458, 459], vec![460, 462, 463]]); + assert_eq!( + siblings, + vec![ + RecursiveIndex { left_indexes: vec![], right_indexes: vec![441, 442, 443] }, + RecursiveIndex { left_indexes: vec![456], right_indexes: vec![458, 459] }, + RecursiveIndex { left_indexes: vec![460], right_indexes: vec![462, 463] } + ] + ); let gmst = GlobalMst::new(GlobalConfig { num_of_tokens: 100, @@ -186,6 +347,12 @@ pub mod test { let global_index = 20; let siblings = get_recursive_siblings_index(global_index, &gmst); - assert_eq!(siblings, vec![[40, 42, 43], [44, 46, 47]]); + assert_eq!( + siblings, + vec![ + RecursiveIndex { left_indexes: vec![40], right_indexes: vec![42, 43] }, + RecursiveIndex { left_indexes: vec![44], right_indexes: vec![46, 47] }, + ] + ); } } From 1c4ef8eb5fdc23429904bc86da96304748dab1a8 Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Fri, 16 Aug 2024 15:55:04 +0800 Subject: [PATCH 02/24] test:test-get-index --- crates/zk-por-core/src/merkle_proof.rs | 43 +++++++++++++++++++++----- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index aa2fa1d..ea44f5d 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -9,7 +9,7 @@ use crate::{ account::Account, database::{DataBase, UserId}, global::GlobalMst, - merkle_sum_prover::utils::{hash_2_subhashes, hash_inputs}, + merkle_sum_prover::utils::hash_2_subhashes, types::{D, F}, }; @@ -138,12 +138,6 @@ pub fn get_recursive_siblings_index( siblings } -/// Hashes for a given users merkle proof of inclusion siblings in the Global Merkle Sum Tree -#[derive(Debug, Clone)] -pub struct MerkleProof { - pub sum_tree_siblings: Vec>, - pub recursive_tree_siblings: Vec, -} /// We use this wrapper struct for the left and right hashes of our recursive siblings. This is needed so a user knows the position of /// their own hash when hashing. @@ -179,6 +173,13 @@ impl RecursiveHashes { } } +/// Hashes for a given users merkle proof of inclusion siblings in the Global Merkle Sum Tree +#[derive(Debug, Clone)] +pub struct MerkleProof { + pub sum_tree_siblings: Vec>, + pub recursive_tree_siblings: Vec, +} + impl MerkleProof { pub fn new_from_user_id(user_id: UserId, db: &DataBase, global_mst: &GlobalMst) -> MerkleProof { let user_index = db.get_user_index(user_id); @@ -229,6 +230,7 @@ impl MerkleProof { } } + /// Given the indexes for the MST siblings, get the hashes from the database for the merkle proof of inclusion. pub fn get_merkle_proof_hashes_from_indexes( indexes: &MerkleProofIndex, @@ -253,7 +255,7 @@ pub fn get_merkle_proof_hashes_from_indexes( pub mod test { use crate::{ global::{GlobalConfig, GlobalMst}, - merkle_proof::{get_recursive_siblings_index, RecursiveIndex}, + merkle_proof::{get_recursive_siblings_index, MerkleProofIndex, RecursiveIndex}, }; use super::get_mst_siblings_index; @@ -355,4 +357,29 @@ pub mod test { ] ); } + + #[test] + pub fn test_get_new_merkle_index_from_user_index(){ + let gmst = GlobalMst::new(GlobalConfig { + num_of_tokens: 100, + num_of_batches: 15, + batch_size: 4, + recursion_branchout_num: 4, + }); + + let global_index = 0; + + let merkle_proof_indexes = MerkleProofIndex::new_from_user_index(global_index, &gmst); + + assert_eq!( + merkle_proof_indexes, + MerkleProofIndex{ + sum_tree_siblings:vec![1, 61], + recursive_tree_siblings: vec![ + RecursiveIndex { left_indexes: vec![], right_indexes: vec![91, 92, 93] }, + RecursiveIndex { left_indexes: vec![], right_indexes: vec![107, 108, 109] } + ] , + } + ); + } } From c5167d29b7058b27cbb28e2d87286839d8f76d8f Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Fri, 16 Aug 2024 19:09:57 +0800 Subject: [PATCH 03/24] feature:add-merkle-proof-sdk --- config/default.toml | 1 + crates/zk-por-cli/src/lib.rs | 1 + crates/zk-por-cli/src/main.rs | 17 +- crates/zk-por-cli/src/merkle_proof.rs | 53 ++++ crates/zk-por-core/src/config.rs | 17 ++ crates/zk-por-core/src/global.rs | 106 ++++--- crates/zk-por-core/src/merkle_proof.rs | 349 +++++++++++++++++----- crates/zk-por-core/src/merkle_sum_tree.rs | 1 + 8 files changed, 428 insertions(+), 117 deletions(-) create mode 100644 crates/zk-por-cli/src/merkle_proof.rs diff --git a/config/default.toml b/config/default.toml index 74256a7..a0c962d 100644 --- a/config/default.toml +++ b/config/default.toml @@ -3,6 +3,7 @@ round_no = 0 batch_size = 1024 user_data_path = "/opt/data/zkpor/users/" num_of_tokens = 220 +num_users = 2048 [db] level_db_user_path = "level_db_user_data" level_db_gmst_path = "level_db_gmst_data" diff --git a/crates/zk-por-cli/src/lib.rs b/crates/zk-por-cli/src/lib.rs index 7d62afa..9794d33 100644 --- a/crates/zk-por-cli/src/lib.rs +++ b/crates/zk-por-cli/src/lib.rs @@ -1,3 +1,4 @@ mod constant; +pub mod merkle_proof; pub mod prover; pub mod verifier; diff --git a/crates/zk-por-cli/src/main.rs b/crates/zk-por-cli/src/main.rs index aae41d0..73491be 100644 --- a/crates/zk-por-cli/src/main.rs +++ b/crates/zk-por-cli/src/main.rs @@ -1,8 +1,8 @@ use std::{path::PathBuf, str::FromStr}; use clap::{Parser, Subcommand}; -use zk_por_cli::{prover::prove, verifier::verify}; -use zk_por_core::error::PoRError; +use zk_por_cli::{merkle_proof::get_merkle_proof, prover::prove, verifier::verify}; +use zk_por_core::{config::ProverConfig, error::PoRError}; #[derive(Parser)] #[command(version, about, long_about = None)] @@ -24,8 +24,12 @@ pub enum ZkPorCommitCommands { output_path: String, // path to output file }, GetMerkleProof { + #[arg(short, long)] + cfg_path: String, // path to config file #[arg(short, long)] user_id: String, + #[arg(short, long)] + output_path: String, // path to output file }, Verify { #[arg(short, long)] @@ -45,10 +49,11 @@ impl Execute for ZkPorCommitCommands { let output_file = PathBuf::from_str(&output_path).unwrap(); prove(prover_cfg, output_file) } - ZkPorCommitCommands::GetMerkleProof { user_id } => { - // TODO: implement this - _ = user_id; - Ok(()) + ZkPorCommitCommands::GetMerkleProof { cfg_path, user_id, output_path } => { + let cfg = zk_por_core::config::ProverConfig::load(&cfg_path) + .map_err(|e| PoRError::ConfigError(e))?; + let prover_cfg: ProverConfig = cfg.try_deserialize().unwrap(); + get_merkle_proof(user_id.to_string(), prover_cfg, output_path.to_string()) } ZkPorCommitCommands::Verify { global_proof_path, inclusion_proof_path } => { let global_proof_path = PathBuf::from_str(&global_proof_path).unwrap(); diff --git a/crates/zk-por-cli/src/merkle_proof.rs b/crates/zk-por-cli/src/merkle_proof.rs new file mode 100644 index 0000000..dce18ca --- /dev/null +++ b/crates/zk-por-cli/src/merkle_proof.rs @@ -0,0 +1,53 @@ +use std::{fs::File, io::Write, str::FromStr}; + +use serde_json::json; +use zk_por_core::{ + config::ProverConfig, + database::{DataBase, DbOption}, + error::PoRError, + global::GlobalConfig, + merkle_proof::MerkleProof, + parser::{AccountParser, FilesCfg, FilesParser}, +}; + +use crate::constant::RECURSION_BRANCHOUT_NUM; + +pub fn get_merkle_proof( + user_id: String, + cfg: ProverConfig, + output_path: String, +) -> Result<(), PoRError> { + let database = DataBase::new(DbOption { + user_map_dir: cfg.db.level_db_user_path.to_string(), + gmst_dir: cfg.db.level_db_gmst_path.to_string(), + }); + + let batch_size = cfg.prover.batch_size as usize; + let token_num = cfg.prover.num_of_tokens as usize; + + // the path to dump the final generated proof + let parser = FilesParser::new(FilesCfg { + dir: std::path::PathBuf::from_str(&cfg.prover.user_data_path).unwrap(), + batch_size: cfg.prover.batch_size, + num_of_tokens: cfg.prover.num_of_tokens, + }); + parser.log_state(); + let account_parser: Box = Box::new(parser); + + let batch_num = account_parser.total_num_of_users().div_ceil(batch_size); + + let global_cfg = GlobalConfig { + num_of_tokens: token_num, + num_of_batches: batch_num, + batch_size: batch_size, + recursion_branchout_num: RECURSION_BRANCHOUT_NUM, + }; + + let merkle_proof = MerkleProof::new_from_user_id(user_id, &database, &global_cfg); + let mut file = File::create(output_path.clone()) + .expect(format!("fail to create proof file at {:#?}", output_path).as_str()); + file.write_all(json!(merkle_proof).to_string().as_bytes()) + .expect("fail to write proof to file"); + + Ok(()) +} diff --git a/crates/zk-por-core/src/config.rs b/crates/zk-por-core/src/config.rs index 5180c70..c3651f2 100644 --- a/crates/zk-por-core/src/config.rs +++ b/crates/zk-por-core/src/config.rs @@ -32,6 +32,7 @@ pub struct ConfigProver { pub batch_size: usize, pub num_of_tokens: usize, pub user_data_path: String, + pub num_users: usize, } #[derive(Debug, Clone, Deserialize)] @@ -40,6 +41,22 @@ pub struct ConfigDb { pub level_db_gmst_path: String, } +impl ConfigDb { + pub fn load(dir: &str) -> Result { + let env = std::env::var("ENV").unwrap_or("default".into()); + Config::builder() + // .add_source(File::with_name(&format!("{}/default", dir))) + .add_source(File::with_name(&format!("{}/{}", dir, env)).required(false)) + .add_source(File::with_name(&format!("{}/local", dir)).required(false)) + .add_source(config::Environment::with_prefix("ZKPOR")) + .build() + } + pub fn try_new() -> Result { + let config = Self::load("config")?; + config.try_deserialize() + } +} + #[derive(Debug, Clone, Deserialize)] pub struct ProverConfig { pub log: ConfigLog, diff --git a/crates/zk-por-core/src/global.rs b/crates/zk-por-core/src/global.rs index 1a198d4..b71c76d 100644 --- a/crates/zk-por-core/src/global.rs +++ b/crates/zk-por-core/src/global.rs @@ -10,7 +10,7 @@ use plonky2::{hash::hash_types::HashOut, util::log2_strict}; use std::{ops::Div, sync::RwLock}; use tracing::debug; -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub struct GlobalConfig { pub num_of_tokens: usize, pub num_of_batches: usize, @@ -34,7 +34,7 @@ impl GlobalMst { let mst_vec = vec![HashOut::default(); 0]; // will resize later let mut mst = Self { inner: mst_vec, top_recursion_level: top_level, cfg: cfg }; // the number of hash is one smaller to the index of the root node of the last recursion level. - let root_node_idx = mst.get_recursive_global_index(top_level, 0); + let root_node_idx = GlobalMst::get_recursive_global_index(&cfg, top_level, 0); let tree_size = root_node_idx + 1; mst.inner.resize(tree_size, HashOut::default()); mst @@ -44,8 +44,8 @@ impl GlobalMst { self.inner.len() } - pub fn get_num_of_leaves(&self) -> usize { - self.cfg.batch_size * self.cfg.num_of_batches + pub fn get_num_of_leaves(cfg: &GlobalConfig) -> usize { + cfg.batch_size * cfg.num_of_batches } pub fn get_nodes(&self, range: std::ops::Range) -> &[HashOut] { @@ -58,14 +58,18 @@ impl GlobalMst { /// 12 13 /// 8-9, 10-11 /// 0 - 3, 4 - 7 - pub fn get_batch_tree_global_index(&self, batch_idx: usize, inner_tree_idx: usize) -> usize { - let batch_size = self.cfg.batch_size; + pub fn get_batch_tree_global_index( + cfg: &GlobalConfig, + batch_idx: usize, + inner_tree_idx: usize, + ) -> usize { + let batch_size = cfg.batch_size; let tree_depth = log2_strict(batch_size); let batch_tree_level = get_node_level(batch_size, inner_tree_idx); let level_from_bottom = tree_depth - batch_tree_level; - let numeritor = 2 * batch_size * self.cfg.num_of_batches; + let numeritor = 2 * batch_size * cfg.num_of_batches; let global_tree_vertical_offset = numeritor - numeritor.div(1 << level_from_bottom); // the gmst idx of the first node at {level_from_bottom} level let level_node_counts = batch_size.div(1 << level_from_bottom); @@ -83,21 +87,21 @@ impl GlobalMst { // mst root node at level 0, pub fn get_recursive_global_index( - &self, + cfg: &GlobalConfig, recursive_level: usize, inner_level_idx: usize, ) -> usize { - let mst_node_num = 2 * self.cfg.batch_size - 1; - let batch_num = self.cfg.num_of_batches; - let branchout_num = self.cfg.recursion_branchout_num; + let mst_node_num = 2 * cfg.batch_size - 1; + let batch_num = cfg.num_of_batches; + let branchout_num = cfg.recursion_branchout_num; if recursive_level == 0 { // level of merkle sum tree root - if inner_level_idx < self.cfg.num_of_batches { + if inner_level_idx < cfg.num_of_batches { // the global index of the root of the batch tree let mst_root_idx = mst_node_num - 1; - return self.get_batch_tree_global_index(inner_level_idx, mst_root_idx); + return GlobalMst::get_batch_tree_global_index(cfg, inner_level_idx, mst_root_idx); } else { - return batch_num * mst_node_num + (inner_level_idx - self.cfg.num_of_batches); + return batch_num * mst_node_num + (inner_level_idx - cfg.num_of_batches); } } @@ -115,9 +119,9 @@ impl GlobalMst { let mut level = recursive_level; while level > 1 { - let mut this_level_node_num = last_level_node_num / self.cfg.recursion_branchout_num; + let mut this_level_node_num = last_level_node_num / cfg.recursion_branchout_num; this_level_node_num = - pad_to_multiple_of(this_level_node_num, self.cfg.recursion_branchout_num); + pad_to_multiple_of(this_level_node_num, cfg.recursion_branchout_num); recursive_offset += this_level_node_num; @@ -132,14 +136,18 @@ impl GlobalMst { /// `batch_idx`: index indicating the batch index /// `i`: the sub batch tree index; e.g the batch tree is of size 1<<10; i \in [0, 2*batch_size) pub fn set_batch_hash(&mut self, batch_idx: usize, i: usize, hash: HashOut) { - let global_mst_idx = self.get_batch_tree_global_index(batch_idx, i); + let global_mst_idx = GlobalMst::get_batch_tree_global_index(&self.cfg, batch_idx, i); self.inner[global_mst_idx] = hash; } pub fn get_batch_root_hash(&self, batch_idx: usize) -> HashOut { debug!("get batch root hash, batch_idx: {:?}", batch_idx); assert!(batch_idx < self.cfg.num_of_batches); - let root_idx = self.get_batch_tree_global_index(batch_idx, 2 * self.cfg.batch_size - 2); + let root_idx = GlobalMst::get_batch_tree_global_index( + &self.cfg, + batch_idx, + 2 * self.cfg.batch_size - 2, + ); self.inner[root_idx] } @@ -149,7 +157,7 @@ impl GlobalMst { "set_recursive_hash, recursive_level: {:?}, index: {:?}, hash: {:?}", recursive_level, index, hash ); - let idx = self.get_recursive_global_index(recursive_level, index); + let idx = GlobalMst::get_recursive_global_index(&self.cfg, recursive_level, index); self.inner[idx] = hash; } @@ -164,11 +172,18 @@ impl GlobalMst { let inner_left_child_idx = 2 * (inner_tree_idx - leaf_size); let inner_right_child_idx = 2 * (inner_tree_idx - leaf_size) + 1; - let global_parent_idx = self.get_batch_tree_global_index(tree_idx, inner_tree_idx); - let global_left_child_idx = - self.get_batch_tree_global_index(tree_idx, inner_left_child_idx); - let global_right_child_idx = - self.get_batch_tree_global_index(tree_idx, inner_right_child_idx); + let global_parent_idx = + GlobalMst::get_batch_tree_global_index(&self.cfg, tree_idx, inner_tree_idx); + let global_left_child_idx = GlobalMst::get_batch_tree_global_index( + &self.cfg, + tree_idx, + inner_left_child_idx, + ); + let global_right_child_idx = GlobalMst::get_batch_tree_global_index( + &self.cfg, + tree_idx, + inner_right_child_idx, + ); visited_global_idx[global_left_child_idx] = true; visited_global_idx[global_right_child_idx] = true; @@ -191,11 +206,12 @@ impl GlobalMst { let inner_child_indexes = (0..branchout_num) .map(|i| inner_idx * branchout_num + i) .collect::>(); - let global_idx = self.get_recursive_global_index(level, inner_idx); + let global_idx = GlobalMst::get_recursive_global_index(&self.cfg, level, inner_idx); let global_child_indexes = inner_child_indexes .iter() .map(|&i| { - let child_global_idx = self.get_recursive_global_index(level - 1, i); + let child_global_idx = + GlobalMst::get_recursive_global_index(&self.cfg, level - 1, i); visited_global_idx[child_global_idx] = true; child_global_idx }) @@ -215,7 +231,8 @@ impl GlobalMst { last_level_node_count = pad_to_multiple_of(this_level_node_count, branchout_num); } } - let global_root_idx = self.get_recursive_global_index(self.top_recursion_level, 0); + let global_root_idx = + GlobalMst::get_recursive_global_index(&self.cfg, self.top_recursion_level, 0); visited_global_idx[global_root_idx] = true; visited_global_idx.iter().all(|&v| v) @@ -274,20 +291,20 @@ mod test { assert_eq!(total_len, 97); assert_eq!(gmst.top_recursion_level, 2); - assert_eq!(gmst.get_batch_tree_global_index(0, 1), 1); - assert_eq!(gmst.get_batch_tree_global_index(0, 14), 84); - assert_eq!(gmst.get_batch_tree_global_index(1, 1), 9); - assert_eq!(gmst.get_batch_tree_global_index(1, 14), 85); - assert_eq!(gmst.get_batch_tree_global_index(5, 7), 47); - assert_eq!(gmst.get_batch_tree_global_index(5, 14), 89); - - assert_eq!(gmst.get_recursive_global_index(0, 7), 91); - assert_eq!(gmst.get_recursive_global_index(0, 1), 85); - assert_eq!(gmst.get_recursive_global_index(1, 0), 92); - assert_eq!(gmst.get_recursive_global_index(1, 1), 93); - assert_eq!(gmst.get_recursive_global_index(1, 2), 94); - assert_eq!(gmst.get_recursive_global_index(1, 3), 95); - assert_eq!(gmst.get_recursive_global_index(2, 0), 96); + assert_eq!(GlobalMst::get_batch_tree_global_index(&gmst.cfg, 0, 1), 1); + assert_eq!(GlobalMst::get_batch_tree_global_index(&gmst.cfg, 0, 14), 84); + assert_eq!(GlobalMst::get_batch_tree_global_index(&gmst.cfg, 1, 1), 9); + assert_eq!(GlobalMst::get_batch_tree_global_index(&gmst.cfg, 1, 14), 85); + assert_eq!(GlobalMst::get_batch_tree_global_index(&gmst.cfg, 5, 7), 47); + assert_eq!(GlobalMst::get_batch_tree_global_index(&gmst.cfg, 5, 14), 89); + + assert_eq!(GlobalMst::get_recursive_global_index(&gmst.cfg, 0, 7), 91); + assert_eq!(GlobalMst::get_recursive_global_index(&gmst.cfg, 0, 1), 85); + assert_eq!(GlobalMst::get_recursive_global_index(&gmst.cfg, 1, 0), 92); + assert_eq!(GlobalMst::get_recursive_global_index(&gmst.cfg, 1, 1), 93); + assert_eq!(GlobalMst::get_recursive_global_index(&gmst.cfg, 1, 2), 94); + assert_eq!(GlobalMst::get_recursive_global_index(&gmst.cfg, 1, 3), 95); + assert_eq!(GlobalMst::get_recursive_global_index(&gmst.cfg, 2, 0), 96); } #[test] @@ -331,8 +348,11 @@ mod test { for inner_idx in 0..this_level_node_count { let children_hashes = (0..branchout_num) .map(|i| { - let child_global_idx = gmst - .get_recursive_global_index(level - 1, inner_idx * branchout_num + i); + let child_global_idx = GlobalMst::get_recursive_global_index( + &gmst.cfg, + level - 1, + inner_idx * branchout_num + i, + ); gmst.inner[child_global_idx] }) .collect::>>(); diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index ea44f5d..413e54e 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -4,11 +4,12 @@ use plonky2::{ plonk::config::Hasher, }; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use serde::Serialize; use crate::{ account::Account, database::{DataBase, UserId}, - global::GlobalMst, + global::{GlobalConfig, GlobalMst}, merkle_sum_prover::utils::hash_2_subhashes, types::{D, F}, }; @@ -29,7 +30,7 @@ pub struct MerkleProofIndex { } impl MerkleProofIndex { - pub fn new_from_user_index(user_index: usize, global_mst: &GlobalMst) -> MerkleProofIndex { + pub fn new_from_user_index(user_index: usize, global_mst: &GlobalConfig) -> MerkleProofIndex { let sum_tree_siblings = get_mst_siblings_index(user_index, global_mst); let recursive_tree_siblings = get_recursive_siblings_index(user_index, global_mst); @@ -39,17 +40,17 @@ impl MerkleProofIndex { /// Get the siblings index for the merkle proof of inclusion given a leaf index of a binary merkle sum tree. /// We get the parent index of a leaf using the formula: parent = index / 2 + num_leaves -pub fn get_mst_siblings_index(global_index: usize, global_mst: &GlobalMst) -> Vec { +pub fn get_mst_siblings_index(global_index: usize, cfg: &GlobalConfig) -> Vec { // Make sure our global index is within the number of leaves - assert!(global_index < global_mst.get_num_of_leaves()); + assert!(global_index < GlobalMst::get_num_of_leaves(cfg)); - let batch_idx = global_index / global_mst.cfg.batch_size; + let batch_idx = global_index / cfg.batch_size; let mut siblings = Vec::new(); // This is the index in the local mst tree - let mut local_index = global_index % global_mst.cfg.batch_size; + let mut local_index = global_index % cfg.batch_size; - while local_index < (global_mst.cfg.batch_size * 2 - 2) { + while local_index < (cfg.batch_size * 2 - 2) { if local_index % 2 == 1 { let sibling_index = local_index - 1; siblings.push(sibling_index); @@ -58,71 +59,74 @@ pub fn get_mst_siblings_index(global_index: usize, global_mst: &GlobalMst) -> Ve siblings.push(sibling_index); } - let parent = local_index / 2 + global_mst.cfg.batch_size; + let parent = local_index / 2 + cfg.batch_size; local_index = parent; } - siblings.par_iter().map(|x| global_mst.get_batch_tree_global_index(batch_idx, *x)).collect() + siblings + .par_iter() + .map(|x| GlobalMst::get_batch_tree_global_index(cfg, batch_idx, *x)) + .collect() } /// Gets the recursive siblings indexes (recursive tree is n-ary tree) as a Vec of vecs, each inner vec is one layer of siblings. pub fn get_recursive_siblings_index( global_index: usize, - global_mst: &GlobalMst, + cfg: &GlobalConfig, ) -> Vec { // Make sure our global index is within the number of leaves - assert!(global_index < global_mst.get_num_of_leaves()); + assert!(global_index < GlobalMst::get_num_of_leaves(cfg)); let mut siblings = Vec::new(); - let local_mst_root_index = global_mst.cfg.batch_size * 2 - 2; - let mst_batch_idx = global_index / global_mst.cfg.batch_size; + let local_mst_root_index = cfg.batch_size * 2 - 2; + let mst_batch_idx = global_index / cfg.batch_size; let this_mst_root_idx = - global_mst.get_batch_tree_global_index(mst_batch_idx, local_mst_root_index); + GlobalMst::get_batch_tree_global_index(cfg, mst_batch_idx, local_mst_root_index); - let first_mst_root_idx = global_mst.get_batch_tree_global_index(0, local_mst_root_index); + let first_mst_root_idx = GlobalMst::get_batch_tree_global_index(cfg, 0, local_mst_root_index); assert!(this_mst_root_idx >= first_mst_root_idx); let this_mst_root_offset = this_mst_root_idx - first_mst_root_idx; - let mut recursive_idx = this_mst_root_offset / global_mst.cfg.recursion_branchout_num; - let mut recursive_offset = this_mst_root_offset % global_mst.cfg.recursion_branchout_num; + let mut recursive_idx = this_mst_root_offset / cfg.recursion_branchout_num; + let mut recursive_offset = this_mst_root_offset % cfg.recursion_branchout_num; - let layers = (global_mst.cfg.num_of_batches.next_power_of_two() as f64) - .log(global_mst.cfg.recursion_branchout_num as f64) + let layers = (cfg.num_of_batches.next_power_of_two() as f64) + .log(cfg.recursion_branchout_num as f64) .ceil() as usize; for i in 0..layers { let mut left_layer = Vec::new(); let mut right_layer = Vec::new(); if i == 0 { - for j in 0..global_mst.cfg.recursion_branchout_num { + for j in 0..cfg.recursion_branchout_num { if j < recursive_offset { - let index = first_mst_root_idx - + (global_mst.cfg.recursion_branchout_num * recursive_idx) - + j; + let index = + first_mst_root_idx + (cfg.recursion_branchout_num * recursive_idx) + j; left_layer.push(index); } if j > recursive_offset { - let index = first_mst_root_idx - + (global_mst.cfg.recursion_branchout_num * recursive_idx) - + j; + let index = + first_mst_root_idx + (cfg.recursion_branchout_num * recursive_idx) + j; right_layer.push(index); } } } else { - for j in 0..global_mst.cfg.recursion_branchout_num { + for j in 0..cfg.recursion_branchout_num { if j < recursive_offset { - let index = global_mst.get_recursive_global_index( + let index = GlobalMst::get_recursive_global_index( + cfg, i, - recursive_idx * global_mst.cfg.recursion_branchout_num + j, + recursive_idx * cfg.recursion_branchout_num + j, ); left_layer.push(index); } if j > recursive_offset { - let index = global_mst.get_recursive_global_index( + let index = GlobalMst::get_recursive_global_index( + cfg, i, - recursive_idx * global_mst.cfg.recursion_branchout_num + j, + recursive_idx * cfg.recursion_branchout_num + j, ); right_layer.push(index); } @@ -131,17 +135,16 @@ pub fn get_recursive_siblings_index( siblings.push(RecursiveIndex { left_indexes: left_layer, right_indexes: right_layer }); - recursive_offset = recursive_idx % global_mst.cfg.recursion_branchout_num; - recursive_idx = recursive_idx / global_mst.cfg.recursion_branchout_num; + recursive_offset = recursive_idx % cfg.recursion_branchout_num; + recursive_idx = recursive_idx / cfg.recursion_branchout_num; } siblings } - /// We use this wrapper struct for the left and right hashes of our recursive siblings. This is needed so a user knows the position of /// their own hash when hashing. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize)] pub struct RecursiveHashes { left_hashes: Vec>, right_hashes: Vec>, @@ -174,46 +177,47 @@ impl RecursiveHashes { } /// Hashes for a given users merkle proof of inclusion siblings in the Global Merkle Sum Tree -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct MerkleProof { + pub index: usize, pub sum_tree_siblings: Vec>, pub recursive_tree_siblings: Vec, } impl MerkleProof { - pub fn new_from_user_id(user_id: UserId, db: &DataBase, global_mst: &GlobalMst) -> MerkleProof { + pub fn new_from_user_id( + user_id_string: String, + db: &DataBase, + cfg: &GlobalConfig, + ) -> MerkleProof { + let user_id = UserId::from_hex_string(user_id_string); let user_index = db.get_user_index(user_id); if user_index.is_none() { tracing::error!("User with id: {:?} does not exist", user_id.to_string()); } - let indexes = - MerkleProofIndex::new_from_user_index(user_index.unwrap() as usize, global_mst); - let merkle_proof = get_merkle_proof_hashes_from_indexes(&indexes, db); + let indexes = MerkleProofIndex::new_from_user_index(user_index.unwrap() as usize, cfg); + let merkle_proof = + get_merkle_proof_hashes_from_indexes(&indexes, user_index.unwrap() as usize, db); merkle_proof } pub fn verify_merkle_proof( &self, account: &Account, - db: DataBase, + user_index: usize, gmst_root: HashOut, ) -> Result { let account_hash = account.get_hash(); - let user_index_res = db.get_user_index(UserId::from_hex_string(account.id.clone())); - if user_index_res.is_none() { - tracing::error!("User with id: {:?} does not exist", account.id.to_string()); - } - - let mut user_index = user_index_res.unwrap(); + let mut index = user_index; let calculated_mst_hash = self.sum_tree_siblings.iter().fold(account_hash, |acc, x| { - if user_index % 2 == 0 { - user_index /= 2; - hash_2_subhashes::(x, &acc) - } else { - user_index /= 2; + if index % 2 == 0 { + index /= 2; hash_2_subhashes::(&acc, x) + } else { + index /= 2; + hash_2_subhashes::(x, &acc) } }); @@ -230,10 +234,10 @@ impl MerkleProof { } } - /// Given the indexes for the MST siblings, get the hashes from the database for the merkle proof of inclusion. pub fn get_merkle_proof_hashes_from_indexes( indexes: &MerkleProofIndex, + user_index: usize, db: &DataBase, ) -> MerkleProof { let mst_hashes: Vec> = indexes @@ -248,17 +252,27 @@ pub fn get_merkle_proof_hashes_from_indexes( .map(|x| RecursiveHashes::new_from_index(x, db)) .collect(); - MerkleProof { sum_tree_siblings: mst_hashes, recursive_tree_siblings: recursive_hashes } + MerkleProof { + sum_tree_siblings: mst_hashes, + recursive_tree_siblings: recursive_hashes, + index: user_index, + } } #[cfg(test)] pub mod test { + use itertools::Itertools; + use plonky2::hash::hash_types::HashOut; + use crate::{ + account::Account, global::{GlobalConfig, GlobalMst}, merkle_proof::{get_recursive_siblings_index, MerkleProofIndex, RecursiveIndex}, + types::F, }; + use plonky2_field::types::Field; - use super::get_mst_siblings_index; + use super::{get_mst_siblings_index, MerkleProof, RecursiveHashes}; #[test] pub fn test_get_siblings_index() { @@ -271,7 +285,7 @@ pub mod test { let global_index = 0; - let siblings = get_mst_siblings_index(global_index, &gmst); + let siblings = get_mst_siblings_index(global_index, &gmst.cfg); assert_eq!(siblings, vec![1, 33, 49]); let gmst = GlobalMst::new(GlobalConfig { @@ -283,7 +297,7 @@ pub mod test { let global_index = 0; - let siblings = get_mst_siblings_index(global_index, &gmst); + let siblings = get_mst_siblings_index(global_index, &gmst.cfg); assert_eq!(siblings, vec![1, 65, 97]); let gmst = GlobalMst::new(GlobalConfig { @@ -295,7 +309,7 @@ pub mod test { let global_index = 0; - let siblings = get_mst_siblings_index(global_index, &gmst); + let siblings = get_mst_siblings_index(global_index, &gmst.cfg); assert_eq!(siblings, vec![1, 49, 73]); } @@ -310,7 +324,7 @@ pub mod test { let global_index = 0; - let siblings = get_recursive_siblings_index(global_index, &gmst); + let siblings = get_recursive_siblings_index(global_index, &gmst.cfg); assert_eq!( siblings, @@ -329,7 +343,7 @@ pub mod test { let global_index = 163; - let siblings = get_recursive_siblings_index(global_index, &gmst); + let siblings = get_recursive_siblings_index(global_index, &gmst.cfg); assert_eq!( siblings, vec![ @@ -340,7 +354,7 @@ pub mod test { ); let gmst = GlobalMst::new(GlobalConfig { - num_of_tokens: 100, + num_of_tokens: 10, num_of_batches: 6, batch_size: 4, recursion_branchout_num: 4, @@ -348,7 +362,7 @@ pub mod test { let global_index = 20; - let siblings = get_recursive_siblings_index(global_index, &gmst); + let siblings = get_recursive_siblings_index(global_index, &gmst.cfg); assert_eq!( siblings, vec![ @@ -359,7 +373,7 @@ pub mod test { } #[test] - pub fn test_get_new_merkle_index_from_user_index(){ + pub fn test_get_new_merkle_index_from_user_index() { let gmst = GlobalMst::new(GlobalConfig { num_of_tokens: 100, num_of_batches: 15, @@ -369,17 +383,216 @@ pub mod test { let global_index = 0; - let merkle_proof_indexes = MerkleProofIndex::new_from_user_index(global_index, &gmst); + let merkle_proof_indexes = MerkleProofIndex::new_from_user_index(global_index, &gmst.cfg); assert_eq!( merkle_proof_indexes, - MerkleProofIndex{ - sum_tree_siblings:vec![1, 61], + MerkleProofIndex { + sum_tree_siblings: vec![1, 61], recursive_tree_siblings: vec![ RecursiveIndex { left_indexes: vec![], right_indexes: vec![91, 92, 93] }, RecursiveIndex { left_indexes: vec![], right_indexes: vec![107, 108, 109] } - ] , - } + ], + } + ); + } + + #[test] + pub fn test_verify_merkle_proof() { + let _gmst = GlobalMst::new(GlobalConfig { + num_of_tokens: 3, + num_of_batches: 4, + batch_size: 2, + recursion_branchout_num: 4, + }); + + let sum_tree_siblings = vec![HashOut::from_vec( + vec![ + 7609058119952049295, + 8895839458156070742, + 1052773619972611009, + 6038312163525827182, + ] + .iter() + .map(|x| F::from_canonical_u64(*x)) + .collect::>(), + )]; + + let recursive_tree_siblings = vec![RecursiveHashes { + left_hashes: vec![], + right_hashes: vec![ + HashOut::from_vec( + vec![ + 15026394135096265436, + 13313300609834454638, + 10151802728958521275, + 6200471959130767555, + ] + .iter() + .map(|x| F::from_canonical_u64(*x)) + .collect::>(), + ), + HashOut::from_vec( + vec![ + 2010803994799996791, + 568450490466247075, + 18209684900543488748, + 7678193912819861368, + ] + .iter() + .map(|x| F::from_canonical_u64(*x)) + .collect::>(), + ), + HashOut::from_vec( + vec![ + 13089029781628355232, + 10704046654659337561, + 15794212269117984095, + 15948192230150472783, + ] + .iter() + .map(|x| F::from_canonical_u64(*x)) + .collect::>(), + ), + ], + }]; + + let merkle_proof = MerkleProof { sum_tree_siblings, recursive_tree_siblings, index: 0 }; + + let root = HashOut::from_vec( + vec![ + 10628303359772907103, + 7478459528589413745, + 12007196562137971174, + 2652030368197917032, + ] + .iter() + .map(|x| F::from_canonical_u64(*x)) + .collect::>(), ); + + let equity = vec![3, 3, 3].iter().map(|x| F::from_canonical_u32(*x)).collect_vec(); + let debt = vec![1, 1, 1].iter().map(|x| F::from_canonical_u32(*x)).collect_vec(); + + let res = merkle_proof.verify_merkle_proof( + &Account { + id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33ad".to_string(), + equity: equity.clone(), + debt: debt.clone(), + }, + 0, + root, + ); + + res.unwrap(); } + + // THIS IS THE TEST DATA FOR VERIFY + // #[test] + // pub fn poseidon_hash() { + // let equity = vec![3,3,3,].iter().map(|x| F::from_canonical_u32(*x)).collect_vec(); + // let debt = vec![1,1,1,].iter().map(|x| F::from_canonical_u32(*x)).collect_vec(); + + // let accounts = vec![ + // Account{ + // id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33ad".to_string(), + // equity: equity.clone(), + // debt: debt.clone(), + // }, + // Account{ + // id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33ac".to_string(), + // equity: equity.clone(), + // debt: debt.clone(), + // }, + // Account{ + // id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33ab".to_string(), + // equity: equity.clone(), + // debt: debt.clone(), + // }, + // Account{ + // id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33aa".to_string(), + // equity: equity.clone(), + // debt: debt.clone(), + // }, + // Account{ + // id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33a1".to_string(), + // equity: equity.clone(), + // debt: debt.clone(), + // }, + // Account{ + // id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33a2".to_string(), + // equity: equity.clone(), + // debt: debt.clone(), + // }, + // Account{ + // id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33a3".to_string(), + // equity: equity.clone(), + // debt: debt.clone(), + // }, + // Account{ + // id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33a4".to_string(), + // equity: equity.clone(), + // debt: debt.clone(), + // } + // ]; + + // let msts: Vec = accounts + // .chunks(2) + // .map(|account_batch| MerkleSumTree::new_tree_from_accounts(&account_batch.to_vec())) + // .collect(); + + // let mst_hashes = msts.iter().map(|x| x.merkle_sum_tree.iter().map(|y| y.hash).collect_vec()).collect_vec(); + // println!("msts:{:?}", mst_hashes); + // let inputs = vec![ + // HashOut::from_vec( + // vec![ + // 8699257539652901730, + // 12847577670763395377, + // 14540605839220144846, + // 1921995570040415498, + // ] + // .iter() + // .map(|x| F::from_canonical_u64(*x)) + // .collect::>(), + // ), + // HashOut::from_vec( + // vec![ + // 15026394135096265436, + // 13313300609834454638, + // 10151802728958521275, + // 6200471959130767555, + // ] + // .iter() + // .map(|x| F::from_canonical_u64(*x)) + // .collect::>(), + // ), + // HashOut::from_vec( + // vec![ + // 2010803994799996791, + // 568450490466247075, + // 18209684900543488748, + // 7678193912819861368, + // ] + // .iter() + // .map(|x| F::from_canonical_u64(*x)) + // .collect::>(), + // ), + // HashOut::from_vec( + // vec![ + // 13089029781628355232, + // 10704046654659337561, + // 15794212269117984095, + // 15948192230150472783, + // ] + // .iter() + // .map(|x| F::from_canonical_u64(*x)) + // .collect::>(), + // ), + // ]; + + // let hash = PoseidonHash::hash_no_pad( + // inputs.iter().map(|x| x.elements).flatten().collect_vec().as_slice(), + // ); + // println!("Hash: {:?}", hash); + // } } diff --git a/crates/zk-por-core/src/merkle_sum_tree.rs b/crates/zk-por-core/src/merkle_sum_tree.rs index 9a97403..0ec9173 100644 --- a/crates/zk-por-core/src/merkle_sum_tree.rs +++ b/crates/zk-por-core/src/merkle_sum_tree.rs @@ -36,6 +36,7 @@ impl MerkleSumNode { } /// Struct representing a merkle sum tree, it is represented as a vector of Merkle Sum Nodes. +#[derive(Debug, Clone)] pub struct MerkleSumTree { pub merkle_sum_tree: Vec, pub tree_depth: usize, From 4b855543bd411eb0e24b28696005c1bda85b3c92 Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Fri, 16 Aug 2024 19:14:22 +0800 Subject: [PATCH 04/24] chore:merge-fix --- crates/zk-por-cli/src/merkle_proof.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/zk-por-cli/src/merkle_proof.rs b/crates/zk-por-cli/src/merkle_proof.rs index dce18ca..d6ca4ff 100644 --- a/crates/zk-por-cli/src/merkle_proof.rs +++ b/crates/zk-por-cli/src/merkle_proof.rs @@ -7,7 +7,7 @@ use zk_por_core::{ error::PoRError, global::GlobalConfig, merkle_proof::MerkleProof, - parser::{AccountParser, FilesCfg, FilesParser}, + parser::{AccountParser, FileAccountReader, FileManager, FilesCfg}, }; use crate::constant::RECURSION_BRANCHOUT_NUM; @@ -26,13 +26,17 @@ pub fn get_merkle_proof( let token_num = cfg.prover.num_of_tokens as usize; // the path to dump the final generated proof - let parser = FilesParser::new(FilesCfg { - dir: std::path::PathBuf::from_str(&cfg.prover.user_data_path).unwrap(), - batch_size: cfg.prover.batch_size, - num_of_tokens: cfg.prover.num_of_tokens, - }); - parser.log_state(); - let account_parser: Box = Box::new(parser); + let file_manager = FileManager {}; + let account_parser = FileAccountReader::new( + FilesCfg { + dir: std::path::PathBuf::from_str(&cfg.prover.user_data_path).unwrap(), + batch_size: cfg.prover.batch_size, + num_of_tokens: cfg.prover.num_of_tokens, + }, + &file_manager, + ); + account_parser.log_state(); + // let mut account_parser: Box = Box::new(parser); let batch_num = account_parser.total_num_of_users().div_ceil(batch_size); From 65fc1bebdbc53986f313b14d2dc6bd8df5094823 Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 11:54:53 +0800 Subject: [PATCH 05/24] bugfix: file reader reads zero account --- .gitignore | 3 +- README.md | 13 ++++++ crates/zk-por-core/src/account.rs | 20 +++++---- crates/zk-por-core/src/config.rs | 1 - crates/zk-por-core/src/merkle_proof.rs | 1 + crates/zk-por-core/src/parser.rs | 56 +++++++++++++++++--------- 6 files changed, 66 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 2b18eda..336e7d6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ config/local.toml config/prod.toml test-data/user-data*/ my_permanent_leveldb/ -*level_db*/ \ No newline at end of file +*level_db*/ +*.json \ No newline at end of file diff --git a/README.md b/README.md index 6fa580d..baebf3e 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,12 @@ output_proof_path="global_proof.json" cargo run --release --package zk-por-cli --bin zk-por-cli prove --cfg-path ${cfg_dir_path} --output-path ${output_proof_path} ``` + +- get-merkle-proof +``` +cargo run --release --package zk-por-cli --bin zk-por-cli get-merkle-proof --user-id 409e8a8f5a7e34e4fa13c4a44291e4d5a91bce1a4d90c549765b477fea03fdc5 --output-path merkle_proof.json --cfg-path config +``` + - verify ``` global_root_path="global_proof.json" @@ -45,6 +51,13 @@ arg_inclusion_proof_path="--inclusion-proof-path inclusion_proof.json" cargo run --features zk-por-core/verifier --release --package zk-por-cli --bin zk-por-cli verify --global-proof-path ${global_root_path} ${arg_inclusion_proof_path} ``` +## cli +``` +./target/release/zk-por-cli --help +./target/release/zk-por-cli prove --cfg-path ${cfg_dir_path} --output-path ${output_proof_path} +./target/release/zk-por-cli get-merkle-proof --user-id 409e8a8f5a7e34e4fa13c4a44291e4d5a91bce1a4d90c549765b477fea03fdc5 --output-path merkle_proof.json --cfg-path config +``` + ## code coverage the code test coverage report is auto generated and hosted at [codecov_report](https://okx.github.io/proof-of-reserves-v2/tarpaulin-report.html) diff --git a/crates/zk-por-core/src/account.rs b/crates/zk-por-core/src/account.rs index 5d98838..24c6478 100644 --- a/crates/zk-por-core/src/account.rs +++ b/crates/zk-por-core/src/account.rs @@ -34,6 +34,14 @@ impl Account { hash } + pub fn get_empty_account_with_user_id(user_id: String, num_of_tokens: usize) -> Account { + Self { + id: user_id, + equity: vec![F::default(); num_of_tokens], + debt: vec![F::default(); num_of_tokens], + } + } + pub fn get_empty_account(num_of_tokens: usize) -> Account { Self { id: "0".repeat(64), @@ -70,12 +78,10 @@ pub fn persist_account_id_to_gmst_pos( .iter() .enumerate() .map(|(i, acct)| { - let hex_decode = hex::decode(&acct.id).unwrap(); - assert_eq!(hex_decode.len(), 32); - let mut array = [0u8; 32]; - array.copy_from_slice(&hex_decode); - - (UserId(array), (i + start_idx) as u32) + + let user_id = UserId::from_hex_string(acct.id.to_string()); + tracing::debug!("persist account {:?} with index: {:?}", acct.id, i + start_idx); + (user_id, (i + start_idx) as u32) }) .collect::>(); db.add_batch_users(user_batch); @@ -105,6 +111,6 @@ pub fn gen_accounts_with_random_data(num_accounts: usize, num_assets: usize) -> } pub fn gen_empty_accounts(batch_size: usize, num_assets: usize) -> Vec { - let accounts = vec![Account::get_empty_account(num_assets); batch_size]; + let accounts = vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), num_assets); batch_size]; accounts } diff --git a/crates/zk-por-core/src/config.rs b/crates/zk-por-core/src/config.rs index c3651f2..a40e3e9 100644 --- a/crates/zk-por-core/src/config.rs +++ b/crates/zk-por-core/src/config.rs @@ -32,7 +32,6 @@ pub struct ConfigProver { pub batch_size: usize, pub num_of_tokens: usize, pub user_data_path: String, - pub num_users: usize, } #[derive(Debug, Clone, Deserialize)] diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index 413e54e..b456b90 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -240,6 +240,7 @@ pub fn get_merkle_proof_hashes_from_indexes( user_index: usize, db: &DataBase, ) -> MerkleProof { + // println!("get merkle proof hash for index: {:?}, indexes: {:?}", user_index, indexes); let mst_hashes: Vec> = indexes .sum_tree_siblings .iter() diff --git a/crates/zk-por-core/src/parser.rs b/crates/zk-por-core/src/parser.rs index 4fcf75c..7e7c3dd 100644 --- a/crates/zk-por-core/src/parser.rs +++ b/crates/zk-por-core/src/parser.rs @@ -1,5 +1,5 @@ use super::account::{gen_accounts_with_random_data, Account}; -use crate::types::F; +use crate::{database::UserId, types::F}; use plonky2_field::types::Field; use serde_json::Value; use std::{ @@ -195,7 +195,7 @@ impl AccountParser for FileAccountReader { let mut filled_len = 0; if self.offset < self.buffered_accounts.len() { - let filled_len = self.buffered_accounts.len() - self.offset; + filled_len = self.buffered_accounts.len() - self.offset; result[0..filled_len].clone_from_slice(&self.buffered_accounts[(self.offset)..]); } let mut missing_len = result.len() - (self.buffered_accounts.len() - self.offset); @@ -294,8 +294,7 @@ impl AccountParser for RandomAccountParser { mod test { use crate::{ - account::Account, - parser::{FileManager, FilesCfg}, + account::Account, database::UserId, parser::{FileManager, FilesCfg} }; use mockall::*; use serde_json::Value; @@ -378,14 +377,18 @@ mod test { Ok(paths) }); - mock_file_manager.expect_read_json_into_accounts_vec().times(1).returning(|_| { - let accounts = vec![Account::get_empty_account(20); 4]; - accounts - }); - mock_file_manager.expect_read_json_into_accounts_vec().times(1).returning(|_| { - let accounts = vec![Account::get_empty_account(20); 3]; - accounts - }); + let doc_0: Vec = (0..4).into_iter().map(|_| { + Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20) + }).collect(); + let doc_0_clone = doc_0.clone(); + mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const(doc_0_clone); + + let doc_5: Vec = (0..3).into_iter().map(|_| { + Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20) + }).collect(); + + let doc_5_clone = doc_5.clone(); + mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const(doc_5_clone); let dir = tempdir::TempDir::new("user_input_test").unwrap().into_path(); @@ -395,25 +398,40 @@ mod test { ); assert_eq!(file_acct_reader.total_num_of_users(), 23); - mock_file_manager.expect_read_json_into_accounts_vec().times(1).returning(|_| { - let accounts = vec![Account::get_empty_account(20); 4]; - accounts - }); + let doc_1: Vec = (0..4).into_iter().map(|_| { + Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20) + }).collect(); + + let doc_1_clone = doc_1.clone(); + mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const( doc_1_clone); let users = file_acct_reader.read_n_accounts(0, 8, &mock_file_manager); assert_eq!(users.len(), 8); + assert_eq!(file_acct_reader.file_idx, 1); + assert_eq!(file_acct_reader.offset, 4); + + let expected = vec![doc_0, doc_1].concat(); + expected.iter().zip(users.iter()).for_each(|(expect, actual)| { + assert_eq!(actual.id, expect.id); + }); + mock_file_manager.expect_read_json_into_accounts_vec().times(1).returning(|_| { - let accounts = vec![Account::get_empty_account(20); 4]; + let accounts = vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20); 4]; accounts }); let users = file_acct_reader.read_n_accounts(8, 4, &mock_file_manager); assert_eq!(users.len(), 4); - + assert_eq!(file_acct_reader.file_idx, 2); + assert_eq!(file_acct_reader.offset, 4); mock_file_manager.expect_read_json_into_accounts_vec().times(3).returning(|_| { - let accounts = vec![Account::get_empty_account(20); 4]; + let accounts = vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20); 4]; accounts }); let users = file_acct_reader.read_n_accounts(12, 12, &mock_file_manager); + assert_eq!(file_acct_reader.file_idx, 5); + assert_eq!(file_acct_reader.offset, 3); assert_eq!(users.len(), 11); } + + } From 6e534dff9d4c8e8bb25658b290021601e000d761 Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 11:55:13 +0800 Subject: [PATCH 06/24] fmt --- crates/zk-por-core/src/account.rs | 7 ++++-- crates/zk-por-core/src/parser.rs | 36 +++++++++++++++++-------------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/crates/zk-por-core/src/account.rs b/crates/zk-por-core/src/account.rs index 24c6478..48f88bd 100644 --- a/crates/zk-por-core/src/account.rs +++ b/crates/zk-por-core/src/account.rs @@ -78,7 +78,6 @@ pub fn persist_account_id_to_gmst_pos( .iter() .enumerate() .map(|(i, acct)| { - let user_id = UserId::from_hex_string(acct.id.to_string()); tracing::debug!("persist account {:?} with index: {:?}", acct.id, i + start_idx); (user_id, (i + start_idx) as u32) @@ -111,6 +110,10 @@ pub fn gen_accounts_with_random_data(num_accounts: usize, num_assets: usize) -> } pub fn gen_empty_accounts(batch_size: usize, num_assets: usize) -> Vec { - let accounts = vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), num_assets); batch_size]; + let accounts = + vec![ + Account::get_empty_account_with_user_id(UserId::rand().to_string(), num_assets); + batch_size + ]; accounts } diff --git a/crates/zk-por-core/src/parser.rs b/crates/zk-por-core/src/parser.rs index 7e7c3dd..f2e1c1b 100644 --- a/crates/zk-por-core/src/parser.rs +++ b/crates/zk-por-core/src/parser.rs @@ -294,7 +294,9 @@ impl AccountParser for RandomAccountParser { mod test { use crate::{ - account::Account, database::UserId, parser::{FileManager, FilesCfg} + account::Account, + database::UserId, + parser::{FileManager, FilesCfg}, }; use mockall::*; use serde_json::Value; @@ -377,15 +379,17 @@ mod test { Ok(paths) }); - let doc_0: Vec = (0..4).into_iter().map(|_| { - Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20) - }).collect(); + let doc_0: Vec = (0..4) + .into_iter() + .map(|_| Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20)) + .collect(); let doc_0_clone = doc_0.clone(); mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const(doc_0_clone); - let doc_5: Vec = (0..3).into_iter().map(|_| { - Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20) - }).collect(); + let doc_5: Vec = (0..3) + .into_iter() + .map(|_| Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20)) + .collect(); let doc_5_clone = doc_5.clone(); mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const(doc_5_clone); @@ -398,12 +402,13 @@ mod test { ); assert_eq!(file_acct_reader.total_num_of_users(), 23); - let doc_1: Vec = (0..4).into_iter().map(|_| { - Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20) - }).collect(); + let doc_1: Vec = (0..4) + .into_iter() + .map(|_| Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20)) + .collect(); let doc_1_clone = doc_1.clone(); - mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const( doc_1_clone); + mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const(doc_1_clone); let users = file_acct_reader.read_n_accounts(0, 8, &mock_file_manager); assert_eq!(users.len(), 8); assert_eq!(file_acct_reader.file_idx, 1); @@ -414,9 +419,9 @@ mod test { assert_eq!(actual.id, expect.id); }); - mock_file_manager.expect_read_json_into_accounts_vec().times(1).returning(|_| { - let accounts = vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20); 4]; + let accounts = + vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20); 4]; accounts }); let users = file_acct_reader.read_n_accounts(8, 4, &mock_file_manager); @@ -424,7 +429,8 @@ mod test { assert_eq!(file_acct_reader.file_idx, 2); assert_eq!(file_acct_reader.offset, 4); mock_file_manager.expect_read_json_into_accounts_vec().times(3).returning(|_| { - let accounts = vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20); 4]; + let accounts = + vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20); 4]; accounts }); let users = file_acct_reader.read_n_accounts(12, 12, &mock_file_manager); @@ -432,6 +438,4 @@ mod test { assert_eq!(file_acct_reader.offset, 3); assert_eq!(users.len(), 11); } - - } From e9030d5258432e8859d93529199cda5444a4cbc5 Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 12:06:12 +0800 Subject: [PATCH 07/24] bug fix gmst store index --- crates/zk-por-core/src/account.rs | 2 +- crates/zk-por-core/src/global.rs | 2 +- crates/zk-por-core/src/merkle_proof.rs | 1 - crates/zk-por-core/src/parser.rs | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/zk-por-core/src/account.rs b/crates/zk-por-core/src/account.rs index 48f88bd..76e3fca 100644 --- a/crates/zk-por-core/src/account.rs +++ b/crates/zk-por-core/src/account.rs @@ -79,7 +79,7 @@ pub fn persist_account_id_to_gmst_pos( .enumerate() .map(|(i, acct)| { let user_id = UserId::from_hex_string(acct.id.to_string()); - tracing::debug!("persist account {:?} with index: {:?}", acct.id, i + start_idx); + // tracing::debug!("persist account {:?} with index: {:?}", acct.id, i + start_idx); (user_id, (i + start_idx) as u32) }) .collect::>(); diff --git a/crates/zk-por-core/src/global.rs b/crates/zk-por-core/src/global.rs index b71c76d..21feba0 100644 --- a/crates/zk-por-core/src/global.rs +++ b/crates/zk-por-core/src/global.rs @@ -249,7 +249,7 @@ impl GlobalMst { let batches = (i..end) .into_iter() .enumerate() - .map(|(chunk_idx, j)| ((i + j).try_into().unwrap(), nodes[chunk_idx])) + .map(|(chunk_idx, j)| ((j).try_into().unwrap(), nodes[chunk_idx])) .collect::)>>(); db.add_batch_gmst_nodes(batches); i += chunk_size; diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index b456b90..413e54e 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -240,7 +240,6 @@ pub fn get_merkle_proof_hashes_from_indexes( user_index: usize, db: &DataBase, ) -> MerkleProof { - // println!("get merkle proof hash for index: {:?}, indexes: {:?}", user_index, indexes); let mst_hashes: Vec> = indexes .sum_tree_siblings .iter() diff --git a/crates/zk-por-core/src/parser.rs b/crates/zk-por-core/src/parser.rs index f2e1c1b..3ed8346 100644 --- a/crates/zk-por-core/src/parser.rs +++ b/crates/zk-por-core/src/parser.rs @@ -1,5 +1,5 @@ use super::account::{gen_accounts_with_random_data, Account}; -use crate::{database::UserId, types::F}; +use crate::{ types::F}; use plonky2_field::types::Field; use serde_json::Value; use std::{ From d49ccc1494bab203e9c63fbdda01cf7b05e60039 Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 12:06:26 +0800 Subject: [PATCH 08/24] fmt --- crates/zk-por-core/src/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/zk-por-core/src/parser.rs b/crates/zk-por-core/src/parser.rs index 3ed8346..1eefc0d 100644 --- a/crates/zk-por-core/src/parser.rs +++ b/crates/zk-por-core/src/parser.rs @@ -1,5 +1,5 @@ use super::account::{gen_accounts_with_random_data, Account}; -use crate::{ types::F}; +use crate::types::F; use plonky2_field::types::Field; use serde_json::Value; use std::{ From 9023c4af8559ae59c0a42b99f7b2a7a23fc5d305 Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Mon, 19 Aug 2024 12:16:15 +0800 Subject: [PATCH 09/24] chore:pr-changes --- config/default.toml | 1 - crates/zk-por-cli/src/merkle_proof.rs | 3 +- crates/zk-por-cli/src/verifier.rs | 20 +++++++---- crates/zk-por-core/src/account.rs | 3 +- crates/zk-por-core/src/config.rs | 1 - crates/zk-por-core/src/database.rs | 25 ++++++++------ crates/zk-por-core/src/error.rs | 8 +++++ crates/zk-por-core/src/global.rs | 4 +++ crates/zk-por-core/src/merkle_proof.rs | 47 ++++++++++++++------------ 9 files changed, 69 insertions(+), 43 deletions(-) diff --git a/config/default.toml b/config/default.toml index a0c962d..74256a7 100644 --- a/config/default.toml +++ b/config/default.toml @@ -3,7 +3,6 @@ round_no = 0 batch_size = 1024 user_data_path = "/opt/data/zkpor/users/" num_of_tokens = 220 -num_users = 2048 [db] level_db_user_path = "level_db_user_data" level_db_gmst_path = "level_db_gmst_data" diff --git a/crates/zk-por-cli/src/merkle_proof.rs b/crates/zk-por-cli/src/merkle_proof.rs index d6ca4ff..8e8b6b9 100644 --- a/crates/zk-por-cli/src/merkle_proof.rs +++ b/crates/zk-por-cli/src/merkle_proof.rs @@ -47,7 +47,8 @@ pub fn get_merkle_proof( recursion_branchout_num: RECURSION_BRANCHOUT_NUM, }; - let merkle_proof = MerkleProof::new_from_user_id(user_id, &database, &global_cfg); + let merkle_proof = MerkleProof::new_from_user_id(user_id, &database, &global_cfg).expect("Unable to generate merkle proof"); + let mut file = File::create(output_path.clone()) .expect(format!("fail to create proof file at {:#?}", output_path).as_str()); file.write_all(json!(merkle_proof).to_string().as_bytes()) diff --git a/crates/zk-por-cli/src/verifier.rs b/crates/zk-por-cli/src/verifier.rs index 8da3bf4..b63d42c 100644 --- a/crates/zk-por-cli/src/verifier.rs +++ b/crates/zk-por-cli/src/verifier.rs @@ -1,13 +1,11 @@ +use plonky2::fri::verifier; use serde_json::from_reader; use std::{fs::File, path::PathBuf}; // Assuming Proof is defined in lib.rs and lib.rs is in the same crate use super::constant::RECURSION_BRANCHOUT_NUM; use zk_por_core::{ - circuit_config::{get_recursive_circuit_configs, STANDARD_CONFIG}, - circuit_registry::registry::CircuitRegistry, - error::PoRError, - Proof, + account::Account, circuit_config::{get_recursive_circuit_configs, STANDARD_CONFIG}, circuit_registry::registry::CircuitRegistry, error::PoRError, global::GLOBAL_MST, merkle_proof::MerkleProof, Proof }; pub fn verify( @@ -63,9 +61,19 @@ pub fn verify( } println!("successfully verify the global proof for round {}", round_num); - // TODO: verify the inclusion proof if let Some(merkle_inclusion_path) = merkle_inclusion_path { - _ = merkle_inclusion_path; + let merkle_path = File::open(&merkle_inclusion_path).unwrap(); + let reader = std::io::BufReader::new(merkle_path); + + // Parse the JSON as Proof + let (proof, account): (MerkleProof, Account) = from_reader(reader).unwrap(); + + let global_mst = GLOBAL_MST.get().unwrap(); + let mut _g = global_mst.read().expect("unable to get a lock"); + + let merkle_inlcusion_proof_res = proof.verify_merkle_proof(&account, gmst_root); + + println!("successfully verify the inclusion proof for user for round {}", round_num); } diff --git a/crates/zk-por-core/src/account.rs b/crates/zk-por-core/src/account.rs index 5d98838..83b8c52 100644 --- a/crates/zk-por-core/src/account.rs +++ b/crates/zk-por-core/src/account.rs @@ -3,6 +3,7 @@ use plonky2::{ plonk::config::Hasher, }; use plonky2_field::types::Field; +use serde::{Deserialize, Serialize}; use crate::{ database::{DataBase, UserId}, @@ -11,7 +12,7 @@ use crate::{ use rand::Rng; /// A struct representing a users account. It represents their equity and debt as a Vector of goldilocks field elements. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Account { pub id: String, // 256 bit hex string pub equity: Vec, diff --git a/crates/zk-por-core/src/config.rs b/crates/zk-por-core/src/config.rs index c3651f2..a40e3e9 100644 --- a/crates/zk-por-core/src/config.rs +++ b/crates/zk-por-core/src/config.rs @@ -32,7 +32,6 @@ pub struct ConfigProver { pub batch_size: usize, pub num_of_tokens: usize, pub user_data_path: String, - pub num_users: usize, } #[derive(Debug, Clone, Deserialize)] diff --git a/crates/zk-por-core/src/database.rs b/crates/zk-por-core/src/database.rs index d8cc4c7..d58b84e 100644 --- a/crates/zk-por-core/src/database.rs +++ b/crates/zk-por-core/src/database.rs @@ -5,7 +5,7 @@ use plonky2::{hash::hash_types::HashOut, plonk::config::GenericHashOut}; use rand::Rng; use zk_por_db::LevelDb; -use crate::types::F; +use crate::{error::PoRError, types::F}; #[derive(Debug, Clone, Copy)] pub struct UserId(pub [u8; 32]); @@ -22,21 +22,24 @@ impl UserId { self.0.encode_hex() } - pub fn from_hex_string(hex_str: String) -> Self { + pub fn from_hex_string(hex_str: String) -> Result { if hex_str.len() != 64 { tracing::error!("User Id: {:?} is not a valid id, length is not 256 bits", hex_str); + return Err(PoRError::InvalidParameter(hex_str)); } - let mut arr = [0u8; 32]; - for i in 0..32 { - let byte_str = &hex_str[2 * i..2 * i + 2]; - let byte = u8::from_str_radix(byte_str, 16); - if byte.is_err() { - tracing::error!("User Id: {:?} is not a valid id, not in proper hex form", hex_str); - } - arr[i] = byte.unwrap(); + + let decode_res = hex::decode(hex_str.clone()); + + if decode_res.is_err(){ + tracing::error!("User Id: {:?} is not a valid id", hex_str); + return Err(PoRError::InvalidParameter(hex_str)); } - UserId { 0: arr } + let mut arr = [0u8; 32]; + arr.copy_from_slice(&decode_res.unwrap()); + + + Ok(UserId { 0: arr }) } } diff --git a/crates/zk-por-core/src/error.rs b/crates/zk-por-core/src/error.rs index 24e3c21..cda2d7a 100644 --- a/crates/zk-por-core/src/error.rs +++ b/crates/zk-por-core/src/error.rs @@ -6,6 +6,9 @@ pub enum PoRError { #[error("Proof is not valid")] InvalidProof, + #[error("Merkle proof is not valid")] + InvalidMerkleProof, + #[error("config error: {0}")] ConfigError(#[from] ConfigError), @@ -20,4 +23,9 @@ pub enum PoRError { #[error("The verification circuit digest does not match the prover. ")] CircuitDigestMismatch, + + #[error("User is not valid")] + InvalidUser, + + } diff --git a/crates/zk-por-core/src/global.rs b/crates/zk-por-core/src/global.rs index b71c76d..54872fe 100644 --- a/crates/zk-por-core/src/global.rs +++ b/crates/zk-por-core/src/global.rs @@ -52,6 +52,10 @@ impl GlobalMst { &self.inner[range] } + pub fn get_root(){ + + } + /// convert a mst node inner index to global index in gmst. /// For a mst, the inner index is level-by-level, e.g., /// 14 diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index 413e54e..4d93903 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -4,14 +4,10 @@ use plonky2::{ plonk::config::Hasher, }; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::{ - account::Account, - database::{DataBase, UserId}, - global::{GlobalConfig, GlobalMst}, - merkle_sum_prover::utils::hash_2_subhashes, - types::{D, F}, + account::Account, database::{DataBase, UserId}, error::PoRError, global::{GlobalConfig, GlobalMst}, merkle_sum_prover::utils::hash_2_subhashes, types::{D, F} }; /// We use this wrapper struct for the left and right indexes of our recursive siblings. This is needed so a user knows the position of @@ -40,15 +36,15 @@ impl MerkleProofIndex { /// Get the siblings index for the merkle proof of inclusion given a leaf index of a binary merkle sum tree. /// We get the parent index of a leaf using the formula: parent = index / 2 + num_leaves -pub fn get_mst_siblings_index(global_index: usize, cfg: &GlobalConfig) -> Vec { +pub fn get_mst_siblings_index(global_leaf_index: usize, cfg: &GlobalConfig) -> Vec { // Make sure our global index is within the number of leaves - assert!(global_index < GlobalMst::get_num_of_leaves(cfg)); + assert!(global_leaf_index < GlobalMst::get_num_of_leaves(cfg)); - let batch_idx = global_index / cfg.batch_size; + let batch_idx = global_leaf_index / cfg.batch_size; let mut siblings = Vec::new(); // This is the index in the local mst tree - let mut local_index = global_index % cfg.batch_size; + let mut local_index = global_leaf_index % cfg.batch_size; while local_index < (cfg.batch_size * 2 - 2) { if local_index % 2 == 1 { @@ -144,7 +140,7 @@ pub fn get_recursive_siblings_index( /// We use this wrapper struct for the left and right hashes of our recursive siblings. This is needed so a user knows the position of /// their own hash when hashing. -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RecursiveHashes { left_hashes: Vec>, right_hashes: Vec>, @@ -165,6 +161,7 @@ impl RecursiveHashes { RecursiveHashes { left_hashes, right_hashes } } + /// Left hashes || own hash || Right hashes pub fn get_calculated_hash(self, own_hash: HashOut) -> HashOut { let mut hash_inputs = self.left_hashes; hash_inputs.push(own_hash); @@ -177,7 +174,7 @@ impl RecursiveHashes { } /// Hashes for a given users merkle proof of inclusion siblings in the Global Merkle Sum Tree -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct MerkleProof { pub index: usize, pub sum_tree_siblings: Vec>, @@ -189,27 +186,34 @@ impl MerkleProof { user_id_string: String, db: &DataBase, cfg: &GlobalConfig, - ) -> MerkleProof { - let user_id = UserId::from_hex_string(user_id_string); - let user_index = db.get_user_index(user_id); + ) -> Result { + let user_id_res = UserId::from_hex_string(user_id_string); + if user_id_res.is_err(){ + return Err(user_id_res.unwrap_err()); + } + + let user_id = user_id_res.unwrap(); + + let user_index = db.get_user_index(user_id.clone()); if user_index.is_none() { tracing::error!("User with id: {:?} does not exist", user_id.to_string()); + return Err(PoRError::InvalidParameter(user_id.to_string())) } let indexes = MerkleProofIndex::new_from_user_index(user_index.unwrap() as usize, cfg); let merkle_proof = get_merkle_proof_hashes_from_indexes(&indexes, user_index.unwrap() as usize, db); - merkle_proof + Ok(merkle_proof) } pub fn verify_merkle_proof( &self, account: &Account, - user_index: usize, gmst_root: HashOut, - ) -> Result { + ) -> Result<(), PoRError> { let account_hash = account.get_hash(); - let mut index = user_index; + + let mut index = self.index; let calculated_mst_hash = self.sum_tree_siblings.iter().fold(account_hash, |acc, x| { if index % 2 == 0 { @@ -227,9 +231,9 @@ impl MerkleProof { .fold(calculated_mst_hash, |acc, x| x.clone().get_calculated_hash(acc)); if calculated_hash == gmst_root { - Ok(account.clone()) + Ok(()) } else { - Err("Merkle Proof is not verified".to_string()) + Err(PoRError::InvalidMerkleProof) } } } @@ -480,7 +484,6 @@ pub mod test { equity: equity.clone(), debt: debt.clone(), }, - 0, root, ); From 6ed5dba78b6ee412651bffd3a0555d52d979c51c Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Mon, 19 Aug 2024 12:22:35 +0800 Subject: [PATCH 10/24] feat:add-merkle-verification-to-verifier --- crates/zk-por-cli/src/verifier.rs | 4 ++-- crates/zk-por-core/src/account.rs | 2 +- crates/zk-por-core/src/global.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/zk-por-cli/src/verifier.rs b/crates/zk-por-cli/src/verifier.rs index b63d42c..06e95e3 100644 --- a/crates/zk-por-cli/src/verifier.rs +++ b/crates/zk-por-cli/src/verifier.rs @@ -70,9 +70,9 @@ pub fn verify( let global_mst = GLOBAL_MST.get().unwrap(); let mut _g = global_mst.read().expect("unable to get a lock"); + let gmst_root = _g.inner.last().unwrap(); - let merkle_inlcusion_proof_res = proof.verify_merkle_proof(&account, gmst_root); - + let merkle_inlcusion_proof_res = proof.verify_merkle_proof(&account, *gmst_root).expect("Invalid Merkle Proof"); println!("successfully verify the inclusion proof for user for round {}", round_num); } diff --git a/crates/zk-por-core/src/account.rs b/crates/zk-por-core/src/account.rs index 7682cd7..911a182 100644 --- a/crates/zk-por-core/src/account.rs +++ b/crates/zk-por-core/src/account.rs @@ -79,7 +79,7 @@ pub fn persist_account_id_to_gmst_pos( .iter() .enumerate() .map(|(i, acct)| { - let user_id = UserId::from_hex_string(acct.id.to_string()); + let user_id = UserId::from_hex_string(acct.id.to_string()).unwrap(); // tracing::debug!("persist account {:?} with index: {:?}", acct.id, i + start_idx); (user_id, (i + start_idx) as u32) }) diff --git a/crates/zk-por-core/src/global.rs b/crates/zk-por-core/src/global.rs index 91c913c..8000952 100644 --- a/crates/zk-por-core/src/global.rs +++ b/crates/zk-por-core/src/global.rs @@ -21,7 +21,7 @@ pub struct GlobalConfig { pub static GLOBAL_MST: OnceCell> = OnceCell::new(); pub struct GlobalMst { - inner: Vec>, + pub inner: Vec>, top_recursion_level: usize, pub cfg: GlobalConfig, } @@ -53,7 +53,7 @@ impl GlobalMst { } pub fn get_root(){ - + } /// convert a mst node inner index to global index in gmst. From c3b678dd2ef557fe9e62f74c7454e8ae3c3d4e0e Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Mon, 19 Aug 2024 12:23:45 +0800 Subject: [PATCH 11/24] chore:fmt --- crates/zk-por-cli/src/merkle_proof.rs | 5 +++-- crates/zk-por-cli/src/verifier.rs | 13 ++++++++++--- crates/zk-por-core/src/database.rs | 5 ++--- crates/zk-por-core/src/error.rs | 2 -- crates/zk-por-core/src/global.rs | 4 +--- crates/zk-por-core/src/merkle_proof.rs | 11 ++++++++--- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/crates/zk-por-cli/src/merkle_proof.rs b/crates/zk-por-cli/src/merkle_proof.rs index 8e8b6b9..cf621e6 100644 --- a/crates/zk-por-cli/src/merkle_proof.rs +++ b/crates/zk-por-cli/src/merkle_proof.rs @@ -47,8 +47,9 @@ pub fn get_merkle_proof( recursion_branchout_num: RECURSION_BRANCHOUT_NUM, }; - let merkle_proof = MerkleProof::new_from_user_id(user_id, &database, &global_cfg).expect("Unable to generate merkle proof"); - + let merkle_proof = MerkleProof::new_from_user_id(user_id, &database, &global_cfg) + .expect("Unable to generate merkle proof"); + let mut file = File::create(output_path.clone()) .expect(format!("fail to create proof file at {:#?}", output_path).as_str()); file.write_all(json!(merkle_proof).to_string().as_bytes()) diff --git a/crates/zk-por-cli/src/verifier.rs b/crates/zk-por-cli/src/verifier.rs index 06e95e3..6aa0536 100644 --- a/crates/zk-por-cli/src/verifier.rs +++ b/crates/zk-por-cli/src/verifier.rs @@ -5,7 +5,13 @@ use std::{fs::File, path::PathBuf}; // Assuming Proof is defined in lib.rs and lib.rs is in the same crate use super::constant::RECURSION_BRANCHOUT_NUM; use zk_por_core::{ - account::Account, circuit_config::{get_recursive_circuit_configs, STANDARD_CONFIG}, circuit_registry::registry::CircuitRegistry, error::PoRError, global::GLOBAL_MST, merkle_proof::MerkleProof, Proof + account::Account, + circuit_config::{get_recursive_circuit_configs, STANDARD_CONFIG}, + circuit_registry::registry::CircuitRegistry, + error::PoRError, + global::GLOBAL_MST, + merkle_proof::MerkleProof, + Proof, }; pub fn verify( @@ -64,7 +70,7 @@ pub fn verify( if let Some(merkle_inclusion_path) = merkle_inclusion_path { let merkle_path = File::open(&merkle_inclusion_path).unwrap(); let reader = std::io::BufReader::new(merkle_path); - + // Parse the JSON as Proof let (proof, account): (MerkleProof, Account) = from_reader(reader).unwrap(); @@ -72,7 +78,8 @@ pub fn verify( let mut _g = global_mst.read().expect("unable to get a lock"); let gmst_root = _g.inner.last().unwrap(); - let merkle_inlcusion_proof_res = proof.verify_merkle_proof(&account, *gmst_root).expect("Invalid Merkle Proof"); + let merkle_inlcusion_proof_res = + proof.verify_merkle_proof(&account, *gmst_root).expect("Invalid Merkle Proof"); println!("successfully verify the inclusion proof for user for round {}", round_num); } diff --git a/crates/zk-por-core/src/database.rs b/crates/zk-por-core/src/database.rs index d58b84e..da9fcc9 100644 --- a/crates/zk-por-core/src/database.rs +++ b/crates/zk-por-core/src/database.rs @@ -27,10 +27,10 @@ impl UserId { tracing::error!("User Id: {:?} is not a valid id, length is not 256 bits", hex_str); return Err(PoRError::InvalidParameter(hex_str)); } - + let decode_res = hex::decode(hex_str.clone()); - if decode_res.is_err(){ + if decode_res.is_err() { tracing::error!("User Id: {:?} is not a valid id", hex_str); return Err(PoRError::InvalidParameter(hex_str)); } @@ -38,7 +38,6 @@ impl UserId { let mut arr = [0u8; 32]; arr.copy_from_slice(&decode_res.unwrap()); - Ok(UserId { 0: arr }) } } diff --git a/crates/zk-por-core/src/error.rs b/crates/zk-por-core/src/error.rs index cda2d7a..5f50a49 100644 --- a/crates/zk-por-core/src/error.rs +++ b/crates/zk-por-core/src/error.rs @@ -26,6 +26,4 @@ pub enum PoRError { #[error("User is not valid")] InvalidUser, - - } diff --git a/crates/zk-por-core/src/global.rs b/crates/zk-por-core/src/global.rs index 8000952..8f2126e 100644 --- a/crates/zk-por-core/src/global.rs +++ b/crates/zk-por-core/src/global.rs @@ -52,9 +52,7 @@ impl GlobalMst { &self.inner[range] } - pub fn get_root(){ - - } + pub fn get_root() {} /// convert a mst node inner index to global index in gmst. /// For a mst, the inner index is level-by-level, e.g., diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index 4d93903..934f5fb 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -7,7 +7,12 @@ use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use crate::{ - account::Account, database::{DataBase, UserId}, error::PoRError, global::{GlobalConfig, GlobalMst}, merkle_sum_prover::utils::hash_2_subhashes, types::{D, F} + account::Account, + database::{DataBase, UserId}, + error::PoRError, + global::{GlobalConfig, GlobalMst}, + merkle_sum_prover::utils::hash_2_subhashes, + types::{D, F}, }; /// We use this wrapper struct for the left and right indexes of our recursive siblings. This is needed so a user knows the position of @@ -188,7 +193,7 @@ impl MerkleProof { cfg: &GlobalConfig, ) -> Result { let user_id_res = UserId::from_hex_string(user_id_string); - if user_id_res.is_err(){ + if user_id_res.is_err() { return Err(user_id_res.unwrap_err()); } @@ -197,7 +202,7 @@ impl MerkleProof { let user_index = db.get_user_index(user_id.clone()); if user_index.is_none() { tracing::error!("User with id: {:?} does not exist", user_id.to_string()); - return Err(PoRError::InvalidParameter(user_id.to_string())) + return Err(PoRError::InvalidParameter(user_id.to_string())); } let indexes = MerkleProofIndex::new_from_user_index(user_index.unwrap() as usize, cfg); From 596e1bc6e3d42394ffccaa78505526c9c067803d Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 13:40:10 +0800 Subject: [PATCH 12/24] refactoring readme --- README.md | 4 ++-- crates/zk-por-cli/src/verifier.rs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index baebf3e..50f1d21 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ cargo run --release --package zk-por-cli --bin zk-por-cli get-merkle-proof --use global_root_path="global_proof.json" # optional. If not provided, will skip verifying the inclusion -arg_inclusion_proof_path="--inclusion-proof-path inclusion_proof.json" +inclusion_proof_path="merkle_proof.json" -cargo run --features zk-por-core/verifier --release --package zk-por-cli --bin zk-por-cli verify --global-proof-path ${global_root_path} ${arg_inclusion_proof_path} +cargo run --features zk-por-core/verifier --release --package zk-por-cli --bin zk-por-cli verify --global-proof-path ${global_root_path} --inclusion-proof-path ${inclusion_proof_path} ``` ## cli diff --git a/crates/zk-por-cli/src/verifier.rs b/crates/zk-por-cli/src/verifier.rs index 6aa0536..daad63f 100644 --- a/crates/zk-por-cli/src/verifier.rs +++ b/crates/zk-por-cli/src/verifier.rs @@ -1,4 +1,3 @@ -use plonky2::fri::verifier; use serde_json::from_reader; use std::{fs::File, path::PathBuf}; @@ -78,7 +77,7 @@ pub fn verify( let mut _g = global_mst.read().expect("unable to get a lock"); let gmst_root = _g.inner.last().unwrap(); - let merkle_inlcusion_proof_res = + let _ = proof.verify_merkle_proof(&account, *gmst_root).expect("Invalid Merkle Proof"); println!("successfully verify the inclusion proof for user for round {}", round_num); From fcb183ae6553525dd189156e90ffd9f0effc6f31 Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 14:35:25 +0800 Subject: [PATCH 13/24] fix bug --- crates/zk-por-cli/src/prover.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/zk-por-cli/src/prover.rs b/crates/zk-por-cli/src/prover.rs index 88352fb..144bc1c 100644 --- a/crates/zk-por-cli/src/prover.rs +++ b/crates/zk-por-cli/src/prover.rs @@ -110,7 +110,7 @@ pub fn prove(cfg: ProverConfig, proof_output_path: PathBuf) -> Result<(), PoRErr accounts.resize(account_num + pad_num, Account::get_empty_account(token_num)); } - assert_eq!(account_num % batch_size, 0); + assert_eq!(accounts.len() % batch_size, 0); tracing::debug!( "parse {} times, with number of accounts {}, number of batches {}", From 522edb9e6d09e542ea586948a539a99427c8f1ce Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 14:35:36 +0800 Subject: [PATCH 14/24] fmt --- crates/zk-por-cli/src/verifier.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/zk-por-cli/src/verifier.rs b/crates/zk-por-cli/src/verifier.rs index daad63f..22472d1 100644 --- a/crates/zk-por-cli/src/verifier.rs +++ b/crates/zk-por-cli/src/verifier.rs @@ -77,8 +77,7 @@ pub fn verify( let mut _g = global_mst.read().expect("unable to get a lock"); let gmst_root = _g.inner.last().unwrap(); - let _ = - proof.verify_merkle_proof(&account, *gmst_root).expect("Invalid Merkle Proof"); + let _ = proof.verify_merkle_proof(&account, *gmst_root).expect("Invalid Merkle Proof"); println!("successfully verify the inclusion proof for user for round {}", round_num); } From 354728bc2ee547ebe0d39c8434acdf3a44ec1b66 Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Mon, 19 Aug 2024 14:43:51 +0800 Subject: [PATCH 15/24] chore:small-renaming-changes --- crates/zk-por-cli/src/verifier.rs | 15 ++++++++++----- crates/zk-por-core/src/merkle_proof.rs | 12 ++++++------ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/crates/zk-por-cli/src/verifier.rs b/crates/zk-por-cli/src/verifier.rs index 6aa0536..fed7cb9 100644 --- a/crates/zk-por-cli/src/verifier.rs +++ b/crates/zk-por-cli/src/verifier.rs @@ -1,4 +1,3 @@ -use plonky2::fri::verifier; use serde_json::from_reader; use std::{fs::File, path::PathBuf}; @@ -78,10 +77,16 @@ pub fn verify( let mut _g = global_mst.read().expect("unable to get a lock"); let gmst_root = _g.inner.last().unwrap(); - let merkle_inlcusion_proof_res = - proof.verify_merkle_proof(&account, *gmst_root).expect("Invalid Merkle Proof"); - - println!("successfully verify the inclusion proof for user for round {}", round_num); + let res = + proof.verify_merkle_proof(&account, *gmst_root); + + if res.is_err(){ + let res_err = res.unwrap_err(); + return Err(res_err); + }else{ + println!("successfully verify the inclusion proof for user for round {}", round_num); + return Ok(()) + } } Ok(()) diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index 934f5fb..55705a1 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -45,7 +45,7 @@ pub fn get_mst_siblings_index(global_leaf_index: usize, cfg: &GlobalConfig) -> V // Make sure our global index is within the number of leaves assert!(global_leaf_index < GlobalMst::get_num_of_leaves(cfg)); - let batch_idx = global_leaf_index / cfg.batch_size; + let batch_id = global_leaf_index / cfg.batch_size; let mut siblings = Vec::new(); // This is the index in the local mst tree @@ -60,13 +60,13 @@ pub fn get_mst_siblings_index(global_leaf_index: usize, cfg: &GlobalConfig) -> V siblings.push(sibling_index); } - let parent = local_index / 2 + cfg.batch_size; - local_index = parent; + let local_parent_index = local_index / 2 + cfg.batch_size; + local_index = local_parent_index; } siblings .par_iter() - .map(|x| GlobalMst::get_batch_tree_global_index(cfg, batch_idx, *x)) + .map(|x| GlobalMst::get_batch_tree_global_index(cfg, batch_id, *x)) .collect() } @@ -205,9 +205,9 @@ impl MerkleProof { return Err(PoRError::InvalidParameter(user_id.to_string())); } - let indexes = MerkleProofIndex::new_from_user_index(user_index.unwrap() as usize, cfg); + let merkle_proof_indexes = MerkleProofIndex::new_from_user_index(user_index.unwrap() as usize, cfg); let merkle_proof = - get_merkle_proof_hashes_from_indexes(&indexes, user_index.unwrap() as usize, db); + get_merkle_proof_hashes_from_indexes(&merkle_proof_indexes, user_index.unwrap() as usize, db); Ok(merkle_proof) } From 500e8d96468b8bbd81089ac190aa2d9e401e8cbb Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 14:50:50 +0800 Subject: [PATCH 16/24] add doc --- doc/solution.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/solution.md diff --git a/doc/solution.md b/doc/solution.md new file mode 100644 index 0000000..ed9048f --- /dev/null +++ b/doc/solution.md @@ -0,0 +1,9 @@ +# gmst + +```mermaid + graph TD; + A-->B; + A-->C; + B-->D; + C-->D; +``` \ No newline at end of file From 53a8f578f55a6426ecd2d81c534330befca7469a Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Mon, 19 Aug 2024 15:04:06 +0800 Subject: [PATCH 17/24] feature:add-account-to-merkle-proof --- crates/zk-por-cli/src/main.rs | 6 +-- crates/zk-por-cli/src/merkle_proof.rs | 17 +++++---- crates/zk-por-cli/src/verifier.rs | 13 +++---- crates/zk-por-core/src/account.rs | 22 ++++++++++- crates/zk-por-core/src/merkle_proof.rs | 53 +++++++++++++++----------- 5 files changed, 68 insertions(+), 43 deletions(-) diff --git a/crates/zk-por-cli/src/main.rs b/crates/zk-por-cli/src/main.rs index 73491be..266fead 100644 --- a/crates/zk-por-cli/src/main.rs +++ b/crates/zk-por-cli/src/main.rs @@ -27,7 +27,7 @@ pub enum ZkPorCommitCommands { #[arg(short, long)] cfg_path: String, // path to config file #[arg(short, long)] - user_id: String, + account_path: String, #[arg(short, long)] output_path: String, // path to output file }, @@ -49,11 +49,11 @@ impl Execute for ZkPorCommitCommands { let output_file = PathBuf::from_str(&output_path).unwrap(); prove(prover_cfg, output_file) } - ZkPorCommitCommands::GetMerkleProof { cfg_path, user_id, output_path } => { + ZkPorCommitCommands::GetMerkleProof { cfg_path, account_path, output_path } => { let cfg = zk_por_core::config::ProverConfig::load(&cfg_path) .map_err(|e| PoRError::ConfigError(e))?; let prover_cfg: ProverConfig = cfg.try_deserialize().unwrap(); - get_merkle_proof(user_id.to_string(), prover_cfg, output_path.to_string()) + get_merkle_proof(account_path.to_string(), prover_cfg, output_path.to_string()) } ZkPorCommitCommands::Verify { global_proof_path, inclusion_proof_path } => { let global_proof_path = PathBuf::from_str(&global_proof_path).unwrap(); diff --git a/crates/zk-por-cli/src/merkle_proof.rs b/crates/zk-por-cli/src/merkle_proof.rs index cf621e6..5ed0cce 100644 --- a/crates/zk-por-cli/src/merkle_proof.rs +++ b/crates/zk-por-cli/src/merkle_proof.rs @@ -2,18 +2,13 @@ use std::{fs::File, io::Write, str::FromStr}; use serde_json::json; use zk_por_core::{ - config::ProverConfig, - database::{DataBase, DbOption}, - error::PoRError, - global::GlobalConfig, - merkle_proof::MerkleProof, - parser::{AccountParser, FileAccountReader, FileManager, FilesCfg}, + account::Account, config::ProverConfig, database::{DataBase, DbOption}, error::PoRError, global::GlobalConfig, merkle_proof::MerkleProof, parser::{AccountParser, FileAccountReader, FileManager, FilesCfg} }; use crate::constant::RECURSION_BRANCHOUT_NUM; pub fn get_merkle_proof( - user_id: String, + account_path: String, cfg: ProverConfig, output_path: String, ) -> Result<(), PoRError> { @@ -47,7 +42,13 @@ pub fn get_merkle_proof( recursion_branchout_num: RECURSION_BRANCHOUT_NUM, }; - let merkle_proof = MerkleProof::new_from_user_id(user_id, &database, &global_cfg) + let account = Account::new_from_file_path(account_path); + + if account.is_err(){ + return Err(account.unwrap_err()); + } + + let merkle_proof = MerkleProof::new_from_account(&account.unwrap(), &database, &global_cfg) .expect("Unable to generate merkle proof"); let mut file = File::create(output_path.clone()) diff --git a/crates/zk-por-cli/src/verifier.rs b/crates/zk-por-cli/src/verifier.rs index fed7cb9..04563f6 100644 --- a/crates/zk-por-cli/src/verifier.rs +++ b/crates/zk-por-cli/src/verifier.rs @@ -77,15 +77,14 @@ pub fn verify( let mut _g = global_mst.read().expect("unable to get a lock"); let gmst_root = _g.inner.last().unwrap(); - let res = - proof.verify_merkle_proof(&account, *gmst_root); - - if res.is_err(){ + let res = proof.verify_merkle_proof(&account, *gmst_root); + + if res.is_err() { let res_err = res.unwrap_err(); return Err(res_err); - }else{ - println!("successfully verify the inclusion proof for user for round {}", round_num); - return Ok(()) + } else { + println!("successfully verify the inclusion proof for user for round {}", round_num); + return Ok(()); } } diff --git a/crates/zk-por-core/src/account.rs b/crates/zk-por-core/src/account.rs index 911a182..6dd9192 100644 --- a/crates/zk-por-core/src/account.rs +++ b/crates/zk-por-core/src/account.rs @@ -1,3 +1,4 @@ +use std::{fs::File, io::BufReader}; use plonky2::{ hash::{hash_types::HashOut, poseidon::PoseidonHash}, plonk::config::Hasher, @@ -6,8 +7,7 @@ use plonky2_field::types::Field; use serde::{Deserialize, Serialize}; use crate::{ - database::{DataBase, UserId}, - types::F, + database::{DataBase, UserId}, error::PoRError, types::F }; use rand::Rng; @@ -68,6 +68,24 @@ impl Account { .map(|seg| F::from_canonical_u64(u64::from_str_radix(seg, 16).unwrap())) .collect::>() } + + pub fn new_from_file_path(path: String)-> Result{ + let file_res = File::open(path); + + if file_res.is_err(){ + return Err(PoRError::InvalidParameter("Invalid account json file path".to_string())); + } + + let reader = BufReader::new(file_res.unwrap()); + // Deserialize the json data to a struct + let account_res: Result = serde_json::from_reader(reader); + + if account_res.is_err(){ + return Err(PoRError::InvalidParameter("Invalid account json".to_string())); + } + + Ok(account_res.unwrap()) + } } pub fn persist_account_id_to_gmst_pos( diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index 55705a1..b94e41b 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -64,10 +64,7 @@ pub fn get_mst_siblings_index(global_leaf_index: usize, cfg: &GlobalConfig) -> V local_index = local_parent_index; } - siblings - .par_iter() - .map(|x| GlobalMst::get_batch_tree_global_index(cfg, batch_id, *x)) - .collect() + siblings.par_iter().map(|x| GlobalMst::get_batch_tree_global_index(cfg, batch_id, *x)).collect() } /// Gets the recursive siblings indexes (recursive tree is n-ary tree) as a Vec of vecs, each inner vec is one layer of siblings. @@ -166,7 +163,7 @@ impl RecursiveHashes { RecursiveHashes { left_hashes, right_hashes } } - /// Left hashes || own hash || Right hashes + /// Calculated Hash = Left hashes || own hash || Right hashes pub fn get_calculated_hash(self, own_hash: HashOut) -> HashOut { let mut hash_inputs = self.left_hashes; hash_inputs.push(own_hash); @@ -178,21 +175,23 @@ impl RecursiveHashes { } } -/// Hashes for a given users merkle proof of inclusion siblings in the Global Merkle Sum Tree +/// Hashes for a given users merkle proof of inclusion siblings in the Global Merkle Sum Tree, also includes account data as it is needed for the verification +/// of the merkle proof (needed to calculate own hash) #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MerkleProof { + pub account: Account, pub index: usize, pub sum_tree_siblings: Vec>, pub recursive_tree_siblings: Vec, } impl MerkleProof { - pub fn new_from_user_id( - user_id_string: String, + pub fn new_from_account( + account: &Account, db: &DataBase, cfg: &GlobalConfig, ) -> Result { - let user_id_res = UserId::from_hex_string(user_id_string); + let user_id_res = UserId::from_hex_string(account.id.clone()); if user_id_res.is_err() { return Err(user_id_res.unwrap_err()); } @@ -205,18 +204,22 @@ impl MerkleProof { return Err(PoRError::InvalidParameter(user_id.to_string())); } - let merkle_proof_indexes = MerkleProofIndex::new_from_user_index(user_index.unwrap() as usize, cfg); - let merkle_proof = - get_merkle_proof_hashes_from_indexes(&merkle_proof_indexes, user_index.unwrap() as usize, db); + let merkle_proof_indexes = + MerkleProofIndex::new_from_user_index(user_index.unwrap() as usize, cfg); + let merkle_proof = get_merkle_proof_hashes_from_indexes( + account, + &merkle_proof_indexes, + user_index.unwrap() as usize, + db, + ); Ok(merkle_proof) } pub fn verify_merkle_proof( &self, - account: &Account, gmst_root: HashOut, ) -> Result<(), PoRError> { - let account_hash = account.get_hash(); + let account_hash = self.account.get_hash(); let mut index = self.index; @@ -245,6 +248,7 @@ impl MerkleProof { /// Given the indexes for the MST siblings, get the hashes from the database for the merkle proof of inclusion. pub fn get_merkle_proof_hashes_from_indexes( + account: &Account, indexes: &MerkleProofIndex, user_index: usize, db: &DataBase, @@ -262,6 +266,7 @@ pub fn get_merkle_proof_hashes_from_indexes( .collect(); MerkleProof { + account: account.clone(), sum_tree_siblings: mst_hashes, recursive_tree_siblings: recursive_hashes, index: user_index, @@ -415,6 +420,9 @@ pub mod test { recursion_branchout_num: 4, }); + let equity = vec![3, 3, 3].iter().map(|x| F::from_canonical_u32(*x)).collect_vec(); + let debt = vec![1, 1, 1].iter().map(|x| F::from_canonical_u32(*x)).collect_vec(); + let sum_tree_siblings = vec![HashOut::from_vec( vec![ 7609058119952049295, @@ -466,7 +474,14 @@ pub mod test { ], }]; - let merkle_proof = MerkleProof { sum_tree_siblings, recursive_tree_siblings, index: 0 }; + let account = Account { + id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33ad".to_string(), + equity: equity.clone(), + debt: debt.clone(), + }; + + let merkle_proof = + MerkleProof { account, sum_tree_siblings, recursive_tree_siblings, index: 0 }; let root = HashOut::from_vec( vec![ @@ -480,15 +495,7 @@ pub mod test { .collect::>(), ); - let equity = vec![3, 3, 3].iter().map(|x| F::from_canonical_u32(*x)).collect_vec(); - let debt = vec![1, 1, 1].iter().map(|x| F::from_canonical_u32(*x)).collect_vec(); - let res = merkle_proof.verify_merkle_proof( - &Account { - id: "320b5ea99e653bc2b593db4130d10a4efd3a0b4cc2e1a6672b678d71dfbd33ad".to_string(), - equity: equity.clone(), - debt: debt.clone(), - }, root, ); From 55d4115815f650daded3ab06c5ed337ed60cd9e7 Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Mon, 19 Aug 2024 15:04:21 +0800 Subject: [PATCH 18/24] chore:fmt --- crates/zk-por-cli/src/merkle_proof.rs | 10 ++++++++-- crates/zk-por-core/src/account.rs | 14 ++++++++------ crates/zk-por-core/src/merkle_proof.rs | 9 ++------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/crates/zk-por-cli/src/merkle_proof.rs b/crates/zk-por-cli/src/merkle_proof.rs index 5ed0cce..0029dd4 100644 --- a/crates/zk-por-cli/src/merkle_proof.rs +++ b/crates/zk-por-cli/src/merkle_proof.rs @@ -2,7 +2,13 @@ use std::{fs::File, io::Write, str::FromStr}; use serde_json::json; use zk_por_core::{ - account::Account, config::ProverConfig, database::{DataBase, DbOption}, error::PoRError, global::GlobalConfig, merkle_proof::MerkleProof, parser::{AccountParser, FileAccountReader, FileManager, FilesCfg} + account::Account, + config::ProverConfig, + database::{DataBase, DbOption}, + error::PoRError, + global::GlobalConfig, + merkle_proof::MerkleProof, + parser::{AccountParser, FileAccountReader, FileManager, FilesCfg}, }; use crate::constant::RECURSION_BRANCHOUT_NUM; @@ -44,7 +50,7 @@ pub fn get_merkle_proof( let account = Account::new_from_file_path(account_path); - if account.is_err(){ + if account.is_err() { return Err(account.unwrap_err()); } diff --git a/crates/zk-por-core/src/account.rs b/crates/zk-por-core/src/account.rs index 6dd9192..dc2bb97 100644 --- a/crates/zk-por-core/src/account.rs +++ b/crates/zk-por-core/src/account.rs @@ -1,13 +1,15 @@ -use std::{fs::File, io::BufReader}; use plonky2::{ hash::{hash_types::HashOut, poseidon::PoseidonHash}, plonk::config::Hasher, }; use plonky2_field::types::Field; use serde::{Deserialize, Serialize}; +use std::{fs::File, io::BufReader}; use crate::{ - database::{DataBase, UserId}, error::PoRError, types::F + database::{DataBase, UserId}, + error::PoRError, + types::F, }; use rand::Rng; @@ -69,18 +71,18 @@ impl Account { .collect::>() } - pub fn new_from_file_path(path: String)-> Result{ + pub fn new_from_file_path(path: String) -> Result { let file_res = File::open(path); - if file_res.is_err(){ + if file_res.is_err() { return Err(PoRError::InvalidParameter("Invalid account json file path".to_string())); } let reader = BufReader::new(file_res.unwrap()); // Deserialize the json data to a struct - let account_res: Result = serde_json::from_reader(reader); + let account_res: Result = serde_json::from_reader(reader); - if account_res.is_err(){ + if account_res.is_err() { return Err(PoRError::InvalidParameter("Invalid account json".to_string())); } diff --git a/crates/zk-por-core/src/merkle_proof.rs b/crates/zk-por-core/src/merkle_proof.rs index b94e41b..049aff3 100644 --- a/crates/zk-por-core/src/merkle_proof.rs +++ b/crates/zk-por-core/src/merkle_proof.rs @@ -215,10 +215,7 @@ impl MerkleProof { Ok(merkle_proof) } - pub fn verify_merkle_proof( - &self, - gmst_root: HashOut, - ) -> Result<(), PoRError> { + pub fn verify_merkle_proof(&self, gmst_root: HashOut) -> Result<(), PoRError> { let account_hash = self.account.get_hash(); let mut index = self.index; @@ -495,9 +492,7 @@ pub mod test { .collect::>(), ); - let res = merkle_proof.verify_merkle_proof( - root, - ); + let res = merkle_proof.verify_merkle_proof(root); res.unwrap(); } From 699c3e5448acac0752fc931f1d0fe1110e1fa477 Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 16:14:55 +0800 Subject: [PATCH 19/24] refactor readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50f1d21..e08b81a 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ cargo run --features zk-por-core/verifier --release --package zk-por-cli --bin z ``` ./target/release/zk-por-cli --help ./target/release/zk-por-cli prove --cfg-path ${cfg_dir_path} --output-path ${output_proof_path} -./target/release/zk-por-cli get-merkle-proof --user-id 409e8a8f5a7e34e4fa13c4a44291e4d5a91bce1a4d90c549765b477fea03fdc5 --output-path merkle_proof.json --cfg-path config +./target/release/zk-por-cli get-merkle-proof --account-path account.json --output-path merkle_proof.json --cfg-path config ``` ## code coverage From 7344bbfcdbc5039636608c6fb1f46975e56e907f Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Mon, 19 Aug 2024 16:28:04 +0800 Subject: [PATCH 20/24] change-parser-logic --- crates/zk-por-cli/src/merkle_proof.rs | 1 + crates/zk-por-core/src/account.rs | 16 ++-- crates/zk-por-core/src/parser.rs | 110 +++++++++++--------------- 3 files changed, 57 insertions(+), 70 deletions(-) diff --git a/crates/zk-por-cli/src/merkle_proof.rs b/crates/zk-por-cli/src/merkle_proof.rs index 0029dd4..845cb79 100644 --- a/crates/zk-por-cli/src/merkle_proof.rs +++ b/crates/zk-por-cli/src/merkle_proof.rs @@ -48,6 +48,7 @@ pub fn get_merkle_proof( recursion_branchout_num: RECURSION_BRANCHOUT_NUM, }; + // the account json format is a map of tokens to token values, wheras the format in the merkle proof is given by a vec of token values. let account = Account::new_from_file_path(account_path); if account.is_err() { diff --git a/crates/zk-por-core/src/account.rs b/crates/zk-por-core/src/account.rs index dc2bb97..07c539b 100644 --- a/crates/zk-por-core/src/account.rs +++ b/crates/zk-por-core/src/account.rs @@ -4,12 +4,11 @@ use plonky2::{ }; use plonky2_field::types::Field; use serde::{Deserialize, Serialize}; -use std::{fs::File, io::BufReader}; +use serde_json::Value; +use std::{collections::BTreeMap, fs::File, io::BufReader}; use crate::{ - database::{DataBase, UserId}, - error::PoRError, - types::F, + database::{DataBase, UserId}, error::PoRError, parser::parse_account_state, types::F }; use rand::Rng; @@ -71,6 +70,7 @@ impl Account { .collect::>() } + /// Get a new account from a file path pub fn new_from_file_path(path: String) -> Result { let file_res = File::open(path); @@ -80,13 +80,15 @@ impl Account { let reader = BufReader::new(file_res.unwrap()); // Deserialize the json data to a struct - let account_res: Result = serde_json::from_reader(reader); + let account_map_res: Result, _> = serde_json::from_reader(reader); - if account_res.is_err() { + if account_map_res.is_err() { return Err(PoRError::InvalidParameter("Invalid account json".to_string())); } - Ok(account_res.unwrap()) + let account = parse_account_state(&account_map_res.unwrap()); + + Ok(account) } } diff --git a/crates/zk-por-core/src/parser.rs b/crates/zk-por-core/src/parser.rs index 1eefc0d..ad1af64 100644 --- a/crates/zk-por-core/src/parser.rs +++ b/crates/zk-por-core/src/parser.rs @@ -195,7 +195,7 @@ impl AccountParser for FileAccountReader { let mut filled_len = 0; if self.offset < self.buffered_accounts.len() { - filled_len = self.buffered_accounts.len() - self.offset; + let filled_len = self.buffered_accounts.len() - self.offset; result[0..filled_len].clone_from_slice(&self.buffered_accounts[(self.offset)..]); } let mut missing_len = result.len() - (self.buffered_accounts.len() - self.offset); @@ -233,35 +233,41 @@ impl AccountParser for FileAccountReader { fn parse_exchange_state(parsed_data: &Vec>) -> Vec { let mut accounts_data: Vec = Vec::new(); for obj in parsed_data { - let mut account_id = ""; - let mut inner_vec: Vec = Vec::new(); - for (key, value) in obj.iter() { - if key != "id" { - if let Some(number_str) = value.as_str() { - match number_str.parse::() { - Ok(number) => inner_vec.push(F::from_canonical_u64(number)), - Err(e) => { - error!("Error in parsing token value number: {:?}", e); - panic!("Error in parsing token value number: {:?}", e); - } + accounts_data.push(parse_account_state(obj)); + } + accounts_data +} + + +/// Parses the exchanges state at some snapshot and returns. +pub fn parse_account_state(parsed_data: &BTreeMap) -> Account { + let mut account_id = ""; + let mut inner_vec: Vec = Vec::new(); + for (key, value) in parsed_data.iter() { + if key != "id" { + if let Some(number_str) = value.as_str() { + match number_str.parse::() { + Ok(number) => inner_vec.push(F::from_canonical_u64(number)), + Err(e) => { + error!("Error in parsing token value number: {:?}", e); + panic!("Error in parsing token value number: {:?}", e); } - } else { - error!("Error in parsing string from json: {:?}", value); - panic!("Error in parsing string from json: {:?}", value); } } else { - account_id = value.as_str().unwrap(); + error!("Error in parsing string from json: {:?}", value); + panic!("Error in parsing string from json: {:?}", value); } + } else { + account_id = value.as_str().unwrap(); } - // todo:: currently, we fill debt all to zero - let asset_len = inner_vec.len(); - accounts_data.push(Account { - id: account_id.into(), - equity: inner_vec, - debt: vec![F::ZERO; asset_len], - }); } - accounts_data + // todo:: currently, we fill debt all to zero + let asset_len = inner_vec.len(); + Account { + id: account_id.into(), + equity: inner_vec, + debt: vec![F::ZERO; asset_len], + } } pub struct RandomAccountParser { @@ -295,8 +301,7 @@ mod test { use crate::{ account::Account, - database::UserId, - parser::{FileManager, FilesCfg}, + parser::{parse_exchange_state, FileManager, FilesCfg}, }; use mockall::*; use serde_json::Value; @@ -306,7 +311,7 @@ mod test { str::FromStr, }; - use super::{parse_exchange_state, AccountParser, FileAccountReader, JsonFileManager}; + use super::{AccountParser, FileAccountReader, JsonFileManager}; #[test] pub fn test_read_json_file_into_map() { @@ -379,20 +384,14 @@ mod test { Ok(paths) }); - let doc_0: Vec = (0..4) - .into_iter() - .map(|_| Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20)) - .collect(); - let doc_0_clone = doc_0.clone(); - mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const(doc_0_clone); - - let doc_5: Vec = (0..3) - .into_iter() - .map(|_| Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20)) - .collect(); - - let doc_5_clone = doc_5.clone(); - mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const(doc_5_clone); + mock_file_manager.expect_read_json_into_accounts_vec().times(1).returning(|_| { + let accounts = vec![Account::get_empty_account(20); 4]; + accounts + }); + mock_file_manager.expect_read_json_into_accounts_vec().times(1).returning(|_| { + let accounts = vec![Account::get_empty_account(20); 3]; + accounts + }); let dir = tempdir::TempDir::new("user_input_test").unwrap().into_path(); @@ -402,40 +401,25 @@ mod test { ); assert_eq!(file_acct_reader.total_num_of_users(), 23); - let doc_1: Vec = (0..4) - .into_iter() - .map(|_| Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20)) - .collect(); - - let doc_1_clone = doc_1.clone(); - mock_file_manager.expect_read_json_into_accounts_vec().times(1).return_const(doc_1_clone); + mock_file_manager.expect_read_json_into_accounts_vec().times(1).returning(|_| { + let accounts = vec![Account::get_empty_account(20); 4]; + accounts + }); let users = file_acct_reader.read_n_accounts(0, 8, &mock_file_manager); assert_eq!(users.len(), 8); - assert_eq!(file_acct_reader.file_idx, 1); - assert_eq!(file_acct_reader.offset, 4); - - let expected = vec![doc_0, doc_1].concat(); - expected.iter().zip(users.iter()).for_each(|(expect, actual)| { - assert_eq!(actual.id, expect.id); - }); mock_file_manager.expect_read_json_into_accounts_vec().times(1).returning(|_| { - let accounts = - vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20); 4]; + let accounts = vec![Account::get_empty_account(20); 4]; accounts }); let users = file_acct_reader.read_n_accounts(8, 4, &mock_file_manager); assert_eq!(users.len(), 4); - assert_eq!(file_acct_reader.file_idx, 2); - assert_eq!(file_acct_reader.offset, 4); + mock_file_manager.expect_read_json_into_accounts_vec().times(3).returning(|_| { - let accounts = - vec![Account::get_empty_account_with_user_id(UserId::rand().to_string(), 20); 4]; + let accounts = vec![Account::get_empty_account(20); 4]; accounts }); let users = file_acct_reader.read_n_accounts(12, 12, &mock_file_manager); - assert_eq!(file_acct_reader.file_idx, 5); - assert_eq!(file_acct_reader.offset, 3); assert_eq!(users.len(), 11); } } From 9b5a0af6a04c7339c1537c175b188854fb2fbebc Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Mon, 19 Aug 2024 16:28:46 +0800 Subject: [PATCH 21/24] chore:fmt --- crates/zk-por-core/src/account.rs | 7 +++++-- crates/zk-por-core/src/parser.rs | 7 +------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/zk-por-core/src/account.rs b/crates/zk-por-core/src/account.rs index 07c539b..01462c9 100644 --- a/crates/zk-por-core/src/account.rs +++ b/crates/zk-por-core/src/account.rs @@ -8,7 +8,10 @@ use serde_json::Value; use std::{collections::BTreeMap, fs::File, io::BufReader}; use crate::{ - database::{DataBase, UserId}, error::PoRError, parser::parse_account_state, types::F + database::{DataBase, UserId}, + error::PoRError, + parser::parse_account_state, + types::F, }; use rand::Rng; @@ -87,7 +90,7 @@ impl Account { } let account = parse_account_state(&account_map_res.unwrap()); - + Ok(account) } } diff --git a/crates/zk-por-core/src/parser.rs b/crates/zk-por-core/src/parser.rs index ad1af64..f4ce18b 100644 --- a/crates/zk-por-core/src/parser.rs +++ b/crates/zk-por-core/src/parser.rs @@ -238,7 +238,6 @@ fn parse_exchange_state(parsed_data: &Vec>) -> Vec) -> Account { let mut account_id = ""; @@ -263,11 +262,7 @@ pub fn parse_account_state(parsed_data: &BTreeMap) -> Account { } // todo:: currently, we fill debt all to zero let asset_len = inner_vec.len(); - Account { - id: account_id.into(), - equity: inner_vec, - debt: vec![F::ZERO; asset_len], - } + Account { id: account_id.into(), equity: inner_vec, debt: vec![F::ZERO; asset_len] } } pub struct RandomAccountParser { From fdb2ba9143dd2d75240e2b31edd3855e14d12e79 Mon Sep 17 00:00:00 2001 From: EkamSinghPandher Date: Mon, 19 Aug 2024 17:24:18 +0800 Subject: [PATCH 22/24] refactor:add-root-as-input --- crates/zk-por-cli/src/main.rs | 6 ++++-- crates/zk-por-cli/src/merkle_proof.rs | 2 +- crates/zk-por-cli/src/verifier.rs | 14 ++++++++----- crates/zk-por-core/src/util.rs | 30 ++++++++++++++++++++++++++- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/crates/zk-por-cli/src/main.rs b/crates/zk-por-cli/src/main.rs index 266fead..8e1e26a 100644 --- a/crates/zk-por-cli/src/main.rs +++ b/crates/zk-por-cli/src/main.rs @@ -36,6 +36,8 @@ pub enum ZkPorCommitCommands { global_proof_path: String, #[arg(short, long)] inclusion_proof_path: Option, + #[arg(short, long)] + root: Option, }, } @@ -55,11 +57,11 @@ impl Execute for ZkPorCommitCommands { let prover_cfg: ProverConfig = cfg.try_deserialize().unwrap(); get_merkle_proof(account_path.to_string(), prover_cfg, output_path.to_string()) } - ZkPorCommitCommands::Verify { global_proof_path, inclusion_proof_path } => { + ZkPorCommitCommands::Verify { global_proof_path, inclusion_proof_path, root } => { let global_proof_path = PathBuf::from_str(&global_proof_path).unwrap(); let inclusion_proof_path = inclusion_proof_path.as_ref().map(|p| PathBuf::from_str(&p).unwrap()); - verify(global_proof_path, inclusion_proof_path) + verify(global_proof_path, inclusion_proof_path, root.clone()) } } } diff --git a/crates/zk-por-cli/src/merkle_proof.rs b/crates/zk-por-cli/src/merkle_proof.rs index 845cb79..015b249 100644 --- a/crates/zk-por-cli/src/merkle_proof.rs +++ b/crates/zk-por-cli/src/merkle_proof.rs @@ -56,7 +56,7 @@ pub fn get_merkle_proof( } let merkle_proof = MerkleProof::new_from_account(&account.unwrap(), &database, &global_cfg) - .expect("Unable to generate merkle proof"); + .expect("Una ble to generate merkle proof"); let mut file = File::create(output_path.clone()) .expect(format!("fail to create proof file at {:#?}", output_path).as_str()); diff --git a/crates/zk-por-cli/src/verifier.rs b/crates/zk-por-cli/src/verifier.rs index dccf762..3b7c62b 100644 --- a/crates/zk-por-cli/src/verifier.rs +++ b/crates/zk-por-cli/src/verifier.rs @@ -7,14 +7,16 @@ use zk_por_core::{ circuit_config::{get_recursive_circuit_configs, STANDARD_CONFIG}, circuit_registry::registry::CircuitRegistry, error::PoRError, - global::GLOBAL_MST, merkle_proof::MerkleProof, + types::F, + util::get_hash_from_hash_string, Proof, }; pub fn verify( global_proof_path: PathBuf, merkle_inclusion_path: Option, + root: Option, ) -> Result<(), PoRError> { let proof_file = File::open(&global_proof_path).unwrap(); let reader = std::io::BufReader::new(proof_file); @@ -72,11 +74,13 @@ pub fn verify( // Parse the JSON as Proof let proof: MerkleProof = from_reader(reader).unwrap(); - let global_mst = GLOBAL_MST.get().unwrap(); - let mut _g = global_mst.read().expect("unable to get a lock"); - let gmst_root = _g.inner.last().unwrap(); + if root.is_none() { + return Err(PoRError::InvalidParameter( + "Require root for merkle proof verification".to_string(), + )); + } - let res = proof.verify_merkle_proof(*gmst_root); + let res = proof.verify_merkle_proof(get_hash_from_hash_string(root.unwrap())); if res.is_err() { let res_err = res.unwrap_err(); diff --git a/crates/zk-por-core/src/util.rs b/crates/zk-por-core/src/util.rs index 9ca826e..b216fc3 100644 --- a/crates/zk-por-core/src/util.rs +++ b/crates/zk-por-core/src/util.rs @@ -1,3 +1,7 @@ +use crate::types::F; +use plonky2::hash::hash_types::HashOut; +use plonky2_field::types::Field; + pub fn pad_to_multiple_of(n: usize, multiple: usize) -> usize { if multiple == 0 { return n; // Avoid division by zero @@ -17,11 +21,29 @@ pub fn get_node_level(batch_size: usize, node_idx: usize) -> usize { ((total_nums - node_idx) as f64).log(2.0).floor() as usize } +/// Given a hash string, get a hashout +pub fn get_hash_from_hash_string(hash_string: String) -> HashOut { + let without_brackets = hash_string.trim_matches(|c| c == '[' || c == ']').to_string(); // Remove brackets + + let hash_as_vec_f: Vec = without_brackets + .split(',') + .map(|s| F::from_canonical_u64(s.parse::().unwrap())) + .collect(); + + if hash_as_vec_f.len() != 4 { + panic!("Incorrect format of hash"); + } + + HashOut::from_vec(hash_as_vec_f) +} + #[cfg(test)] pub mod test_util { + use plonky2::hash::hash_types::HashOut; + use crate::util::get_node_level; - use super::pad_to_multiple_of; + use super::{get_hash_from_hash_string, pad_to_multiple_of}; #[test] fn test_get_node_level() { @@ -40,4 +62,10 @@ pub mod test_util { assert_eq!(pad_to_multiple_of(24, 4), 24); assert_eq!(pad_to_multiple_of(27, 4), 28); } + + #[test] + fn test_get_hash_from_hash_string() { + let hash = get_hash_from_hash_string("[0000,0000,0000,0000]".to_string()); + assert_eq!(hash, HashOut::ZERO); + } } From bec9902399987c942cc60b1045041f58ec5c9943 Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 17:43:07 +0800 Subject: [PATCH 23/24] refactor --- README.md | 4 ++-- crates/zk-por-cli/src/prover.rs | 3 +++ crates/zk-por-core/src/global.rs | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e08b81a..6aeaaed 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ cargo run --release --package zk-por-cli --bin zk-por-cli prove --cfg-path ${cfg - get-merkle-proof ``` -cargo run --release --package zk-por-cli --bin zk-por-cli get-merkle-proof --user-id 409e8a8f5a7e34e4fa13c4a44291e4d5a91bce1a4d90c549765b477fea03fdc5 --output-path merkle_proof.json --cfg-path config +cargo run --release --package zk-por-cli --bin zk-por-cli get-merkle-proof --account-path account.json --output-path merkle_proof.json --cfg-path config ``` - verify @@ -48,7 +48,7 @@ global_root_path="global_proof.json" # optional. If not provided, will skip verifying the inclusion inclusion_proof_path="merkle_proof.json" -cargo run --features zk-por-core/verifier --release --package zk-por-cli --bin zk-por-cli verify --global-proof-path ${global_root_path} --inclusion-proof-path ${inclusion_proof_path} +cargo run --features zk-por-core/verifier --release --package zk-por-cli --bin zk-por-cli verify --global-proof-path ${global_root_path} --inclusion-proof-path ${inclusion_proof_path} --root 11288199779358641579,2344540219612146741,6809171731163302525,17936043556479519168 ``` ## cli diff --git a/crates/zk-por-cli/src/prover.rs b/crates/zk-por-cli/src/prover.rs index 144bc1c..4f4183d 100644 --- a/crates/zk-por-cli/src/prover.rs +++ b/crates/zk-por-cli/src/prover.rs @@ -307,7 +307,10 @@ pub fn prove(cfg: ProverConfig, proof_output_path: PathBuf) -> Result<(), PoRErr // persist gmst to database let global_mst = GLOBAL_MST.get().unwrap(); + let _g = global_mst.read().expect("unable to get a lock"); + let root_hash = _g.get_root().expect("no root"); + tracing::info!("root hash is {:?}", root_hash); let start = std::time::Instant::now(); _g.persist(&mut database); tracing::info!("persist gmst to db in {:?}", start.elapsed()); diff --git a/crates/zk-por-core/src/global.rs b/crates/zk-por-core/src/global.rs index 8f2126e..0346cac 100644 --- a/crates/zk-por-core/src/global.rs +++ b/crates/zk-por-core/src/global.rs @@ -52,7 +52,9 @@ impl GlobalMst { &self.inner[range] } - pub fn get_root() {} + pub fn get_root(&self) -> Option<&HashOut> { + self.inner.last() + } /// convert a mst node inner index to global index in gmst. /// For a mst, the inner index is level-by-level, e.g., From 8ac681ae294af8ab142466f00a62c45f96cf2f78 Mon Sep 17 00:00:00 2001 From: cliff0412 Date: Mon, 19 Aug 2024 17:43:43 +0800 Subject: [PATCH 24/24] refactor --- crates/zk-por-cli/src/verifier.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/zk-por-cli/src/verifier.rs b/crates/zk-por-cli/src/verifier.rs index 3b7c62b..d716c2a 100644 --- a/crates/zk-por-cli/src/verifier.rs +++ b/crates/zk-por-cli/src/verifier.rs @@ -8,7 +8,6 @@ use zk_por_core::{ circuit_registry::registry::CircuitRegistry, error::PoRError, merkle_proof::MerkleProof, - types::F, util::get_hash_from_hash_string, Proof, };