From 13e67fc8ed7ed46c82331f7c9af9711da186df74 Mon Sep 17 00:00:00 2001 From: Ratan Kaliani Date: Sat, 12 Aug 2023 17:02:37 -0400 Subject: [PATCH 1/3] integrate variable sha256 --- Cargo.lock | 22 ++++++--- src/step.rs | 11 +++-- src/validator.rs | 125 ++++++++++++----------------------------------- 3 files changed, 52 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a8c2bfd..19ef76a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,9 +137,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.72" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", @@ -830,6 +830,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "dunce" version = "1.0.4" @@ -2006,9 +2012,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "md-5" @@ -2577,12 +2583,13 @@ checksum = "5696e2e2a6bb5c48a6e33fb0dd4d20d0a9472784b709964f337f224e99bd6d06" [[package]] name = "plonky2x" version = "0.1.0" -source = "git+ssh://git@github.com/succinctlabs/sdk.git#d9c89affab6a109650d6049c6ad91266c9230390" +source = "git+ssh://git@github.com/succinctlabs/sdk.git#df0987a3dfc76bbb22d5f594917a595fb496c27c" dependencies = [ "anyhow", "curta", "curve25519-dalek", "digest 0.10.7", + "dotenv", "ethers", "eyre", "hex", @@ -2591,6 +2598,7 @@ dependencies = [ "num", "plonky2", "rand", + "reqwest", "serde", "serde_json", "sha2 0.10.7", @@ -3654,9 +3662,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.30.0" +version = "1.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd" +checksum = "40de3a2ba249dcb097e01be5e67a5ff53cf250397715a071a81543e8a832a920" dependencies = [ "backtrace", "bytes", diff --git a/src/step.rs b/src/step.rs index 856ec5e9..8475d7f2 100644 --- a/src/step.rs +++ b/src/step.rs @@ -18,6 +18,7 @@ use plonky2x::ecc::ed25519::gadgets::curve::CircuitBuilderCurve; use plonky2x::ecc::ed25519::gadgets::eddsa::{EDDSAPublicKeyTarget, EDDSASignatureTarget}; use plonky2x::num::nonnative::nonnative::CircuitBuilderNonNative; use plonky2x::num::u32::gadgets::arithmetic_u32::{CircuitBuilderU32, U32Target}; +use plonky2::iop::target::Target; use crate::signature::TendermintSignature; use crate::utils::{ @@ -35,7 +36,7 @@ pub struct ValidatorTarget { message: ValidatorMessageTarget, message_byte_length: U32Target, voting_power: I64Target, - validator_byte_length: U32Target, + validator_byte_length: Target, enabled: BoolTarget, signed: BoolTarget, } @@ -92,7 +93,7 @@ impl, const D: usize> TendermintStep for Circ let one = self.one(); // Verify each of the validators marshal correctly // Assumes the validators are sorted in the correct order - let byte_lengths: Vec = + let byte_lengths: Vec = validators.iter().map(|v| v.validator_byte_length).collect(); let marshalled_validators: Vec = validators .iter() @@ -248,7 +249,7 @@ where builder.add_virtual_u32_target(), builder.add_virtual_u32_target(), ]); - let validator_byte_length = builder.add_virtual_u32_target(); + let validator_byte_length = builder.add_virtual_target(); let enabled = builder.add_virtual_bool_target_safe(); let signed = builder.add_virtual_bool_target_safe(); @@ -528,9 +529,9 @@ pub(crate) mod tests { ); // Set length targets - pw.set_u32_target( + pw.set_target( celestia_proof_target.validators[i].validator_byte_length, - validator.validator_byte_length as u32, + F::from_canonical_usize(validator.validator_byte_length as usize), ); pw.set_u32_target( celestia_proof_target.validators[i].message_byte_length, diff --git a/src/validator.rs b/src/validator.rs index fa4f825f..a86df8d8 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -13,14 +13,15 @@ use plonky2::{hash::hash_types::RichField, plonk::circuit_builder::CircuitBuilde use plonky2x::ecc::ed25519::curve::curve_types::Curve; use plonky2x::ecc::ed25519::curve::ed25519::Ed25519; use plonky2x::ecc::ed25519::gadgets::curve::{AffinePointTarget, CircuitBuilderCurve}; -use plonky2x::hash::sha::sha256::sha256; +use plonky2x::hash::sha::sha256::{sha256, sha256_variable_length_single_chunk}; use plonky2x::num::u32::gadgets::arithmetic_u32::{CircuitBuilderU32, U32Target}; +use plonky2::iop::target::Target; use crate::utils::{ EncTendermintHashTarget, I64Target, MarshalledValidatorTarget, TendermintHashTarget, HASH_SIZE_BITS, NUM_POSSIBLE_VALIDATOR_BYTE_LENGTHS, PROTOBUF_HASH_SIZE_BITS, VALIDATOR_BYTE_LENGTH_MAX, VALIDATOR_BYTE_LENGTH_MIN, VALIDATOR_SET_SIZE_MAX, - VOTING_POWER_BITS_LENGTH_MAX, VOTING_POWER_BYTES_LENGTH_MAX, + VOTING_POWER_BITS_LENGTH_MAX, VOTING_POWER_BYTES_LENGTH_MAX, VALIDATOR_BIT_LENGTH_MAX, }; pub trait TendermintMarshaller, const D: usize> { @@ -57,14 +58,14 @@ pub trait TendermintMarshaller, const D: usize> { fn hash_validator_leaf( &mut self, validator: &MarshalledValidatorTarget, - validator_byte_length: &U32Target, + validator_byte_length: Target, ) -> TendermintHashTarget; /// Hashes multiple validators to get their leaves according to the Tendermint spec using hash_validator_leaf. fn hash_validator_leaves( &mut self, validators: &Vec, - validator_byte_lengths: &Vec, + validator_byte_lengths: &Vec, ) -> Vec; /// Hashes two nodes to get the inner node according to the Tendermint spec. (0x01 || left || right) @@ -88,7 +89,7 @@ pub trait TendermintMarshaller, const D: usize> { fn hash_validator_set( &mut self, validators: &Vec, - validator_byte_length: &Vec, + validator_byte_lengths: &Vec, validator_enabled: &Vec, ) -> TendermintHashTarget; } @@ -296,85 +297,28 @@ impl, const D: usize> TendermintMarshaller fn hash_validator_leaf( &mut self, validator: &MarshalledValidatorTarget, - validator_byte_length: &U32Target, + validator_byte_length: Target, ) -> TendermintHashTarget { - // Range check the validator byte length is between [VALIDATOR_BYTE_LENGTH_MIN, VALIDATOR_BYTE_LENGTH_MAX] - let min_validator_bytes_length = - self.constant(F::from_canonical_usize(VALIDATOR_BYTE_LENGTH_MIN)); - let max_validator_bytes_length = - self.constant(F::from_canonical_usize(VALIDATOR_BYTE_LENGTH_MAX)); - - // len - min - let diff_with_min_length = self.sub(validator_byte_length.0, min_validator_bytes_length); - - // max - len - let diff_with_max_length = self.sub(max_validator_bytes_length, validator_byte_length.0); - - // Check that diff_with_min_len and diff_with_max_len are both small (if outside of range, one would be a large element of F). - self.range_check(diff_with_min_length, 4); - self.range_check(diff_with_max_length, 4); - - // Note: Because the byte length of each validator is variable, need to hash the validator bytes for each potential byte length. - let mut validator_bytes_hashes = - [[self._false(); HASH_SIZE_BITS]; NUM_POSSIBLE_VALIDATOR_BYTE_LENGTHS]; - for j in 0..NUM_POSSIBLE_VALIDATOR_BYTE_LENGTHS { - // Calculate the length of the message for the leaf hash. - // 0x00 || validatorBytes - let bits_length = 8 + (VALIDATOR_BYTE_LENGTH_MIN + j) * 8; - - // Calculate the message for the leaf hash. - let mut validator_bits = vec![self._false(); bits_length]; - - // 0x00 - for k in 0..8 { - validator_bits[k] = self._false(); - } - - // validatorBytes - for k in 8..bits_length { - validator_bits[k] = validator.0[k - 8]; - } - - // Load the output of the hash. - let hash = sha256(self, &validator_bits); - for k in 0..HASH_SIZE_BITS { - validator_bytes_hashes[j][k] = hash[k]; - } - } - let validator_byte_length_min_constant = - self.constant(F::from_canonical_u32(VALIDATOR_BYTE_LENGTH_MIN as u32)); + let one = self.one(); + let eight = self.constant(F::from_canonical_usize(8)); - // Calculate the index of the validator's bytes length in the range [0, NUM_POSSIBLE_VALIDATOR_BYTE_LENGTHS). - let length_index = self.sub(validator_byte_length.0, validator_byte_length_min_constant); + let enc_validator_byte_length = self.add(one, validator_byte_length); + let enc_validator_bit_length = self.mul(enc_validator_byte_length, eight); - // Create a bitmap, with a selector bit set to 1 if the current index corresponds to the index of the validator's bytes length. - let mut validator_byte_hash_selector = [self._false(); NUM_POSSIBLE_VALIDATOR_BYTE_LENGTHS]; - for j in 0..NUM_POSSIBLE_VALIDATOR_BYTE_LENGTHS { - let byte_length_index = self.constant(F::from_canonical_u32(j as u32)); - validator_byte_hash_selector[j] = self.is_equal(length_index, byte_length_index); + // Encode leaf 0x00 || validator_bits + let mut enc_validator_bits = [self._false(); VALIDATOR_BIT_LENGTH_MAX + 8]; + for i in 0..VALIDATOR_BIT_LENGTH_MAX { + enc_validator_bits[i + 8] = validator.0[i]; } + let hash = sha256_variable_length_single_chunk(self, &enc_validator_bits, enc_validator_bit_length); - let mut ret_validator_leaf_hash = [self._false(); HASH_SIZE_BITS]; - for j in 0..NUM_POSSIBLE_VALIDATOR_BYTE_LENGTHS { - for k in 0..HASH_SIZE_BITS { - // Select the correct byte hash for the validator's byte length. - // Copy the bits from the correct byte hash into the return hash if the selector bit for that byte length is set to 1. - // In all other cases, keep the existing bits in the return hash, yielding desired behavior. - ret_validator_leaf_hash[k] = BoolTarget::new_unsafe(self.select( - validator_byte_hash_selector[j], - validator_bytes_hashes[j][k].target, - ret_validator_leaf_hash[k].target, - )); - } - } - - TendermintHashTarget(ret_validator_leaf_hash) + TendermintHashTarget(hash.try_into().unwrap()) } fn hash_validator_leaves( &mut self, validators: &Vec, - validator_byte_lengths: &Vec, + validator_byte_lengths: &Vec, ) -> Vec { let num_validators = self.constant(F::from_canonical_usize(validators.len())); let num_validator_byte_lengths = @@ -397,7 +341,7 @@ impl, const D: usize> TendermintMarshaller [TendermintHashTarget([self._false(); HASH_SIZE_BITS]); VALIDATOR_SET_SIZE_MAX]; for i in 0..VALIDATOR_SET_SIZE_MAX { validators_leaf_hashes[i] = - self.hash_validator_leaf(&validators[i], &validator_byte_lengths[i]); + self.hash_validator_leaf(&validators[i], validator_byte_lengths[i]); } validators_leaf_hashes.to_vec() @@ -484,7 +428,7 @@ impl, const D: usize> TendermintMarshaller fn hash_validator_set( &mut self, validators: &Vec, - validator_byte_lengths: &Vec, + validator_byte_lengths: &Vec, validator_enabled: &Vec, ) -> TendermintHashTarget { let num_validators = self.constant(F::from_canonical_usize(validators.len())); @@ -573,19 +517,13 @@ pub(crate) mod tests { validators: &Vec, ) -> ( Vec, - Vec, + Vec, Vec, ) { - let mut validator_byte_length: Vec = - vec![ - U32Target(builder.constant(F::from_canonical_usize(VALIDATOR_BYTE_LENGTH_MIN))); - VALIDATOR_SET_SIZE_MAX - ]; + let mut validator_byte_length: Vec = Vec::new(); let mut validator_enabled: Vec = vec![builder._false(); VALIDATOR_SET_SIZE_MAX]; - let mut validator_bits: Vec> = (0..256).map(|_| Vec::::new()).collect(); - let mut validators_target: Vec = vec![ MarshalledValidatorTarget([builder._false(); VALIDATOR_BIT_LENGTH_MAX]); @@ -595,18 +533,17 @@ pub(crate) mod tests { // Convert the hex strings to bytes. for i in 0..validators.len() { let val_byte_length = validators[i].len() / 2; - validator_bits[i] = to_be_bits(hex::decode(&validators[i]).unwrap()); - for j in 0..(val_byte_length * 8) { - if validator_bits[i][j] { - validators_target[i].0[j] = builder._true(); - } else { - validators_target[i].0[j] = builder._false(); - } + let validator_bits = to_be_bits(hex::decode(&validators[i]).unwrap().to_vec()); + for j in 0..validator_bits.len() { + validators_target[i].0[j] = builder.constant_bool(validator_bits[j]); } - validator_byte_length[i] = - U32Target(builder.constant(F::from_canonical_usize(val_byte_length))); + validator_byte_length.push(builder.constant(F::from_canonical_usize(val_byte_length))); validator_enabled[i] = builder._true(); } + + for i in validators.len()..VALIDATOR_SET_SIZE_MAX { + validator_byte_length.push(builder.constant(F::from_canonical_usize(0))); + } return (validators_target, validator_byte_length, validator_enabled); } @@ -765,7 +702,7 @@ pub(crate) mod tests { let (validators_target, validator_byte_length, _) = generate_inputs(&mut builder, &validators); - let result = builder.hash_validator_leaf(&validators_target[0], &validator_byte_length[0]); + let result = builder.hash_validator_leaf(&validators_target[0], validator_byte_length[0]); // Set the target bits to the expected digest bits. for i in 0..HASH_SIZE_BITS { From 639c8b7ba1e770141f5a769e35bc42f01bbb387c Mon Sep 17 00:00:00 2001 From: Ratan Kaliani Date: Sat, 12 Aug 2023 17:23:58 -0400 Subject: [PATCH 2/3] docs --- src/validator.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/validator.rs b/src/validator.rs index a86df8d8..02d9f6c7 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -302,7 +302,9 @@ impl, const D: usize> TendermintMarshaller let one = self.one(); let eight = self.constant(F::from_canonical_usize(8)); + // Add one to account for the 0x00 byte. let enc_validator_byte_length = self.add(one, validator_byte_length); + // Multiply by 8 to get the bit length. let enc_validator_bit_length = self.mul(enc_validator_byte_length, eight); // Encode leaf 0x00 || validator_bits From 2a281cc1b0ecc7d0029f2a7a7d01bf584ee622d3 Mon Sep 17 00:00:00 2001 From: Ratan Kaliani Date: Sat, 12 Aug 2023 17:41:50 -0400 Subject: [PATCH 3/3] cleanup --- src/inputs.rs | 4 ++-- src/validator.rs | 15 ++++----------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/inputs.rs b/src/inputs.rs index 9531da17..f289d47b 100644 --- a/src/inputs.rs +++ b/src/inputs.rs @@ -14,7 +14,7 @@ pub struct Validator { pub message: Vec, pub message_byte_length: u32, pub voting_power: u64, - pub validator_byte_length: u32, + pub validator_byte_length: usize, pub enabled: bool, pub signed: bool, } @@ -106,7 +106,7 @@ pub fn generate_step_inputs(block: usize) -> CelestiaBlockProof { message: signed_vote.sign_bytes(), message_byte_length: signed_vote.sign_bytes().len() as u32, voting_power: validator.power(), - validator_byte_length: val_bytes.len() as u32, + validator_byte_length: val_bytes.len(), enabled: true, signed: true, }); diff --git a/src/validator.rs b/src/validator.rs index 02d9f6c7..76d9d9c4 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -6,7 +6,6 @@ //! The `pubkey` is encoded as the raw list of bytes used in the public key. The `varint` is //! encoded using protobuf's default integer encoding, which consist of 7 bit payloads. You can //! read more about them here: https://protobuf.dev/programming-guides/encoding/#varints. -use curta::plonky2::field::CubicParameters; use plonky2::field::extension::Extendable; use plonky2::iop::target::BoolTarget; use plonky2::{hash::hash_types::RichField, plonk::circuit_builder::CircuitBuilder}; @@ -14,13 +13,13 @@ use plonky2x::ecc::ed25519::curve::curve_types::Curve; use plonky2x::ecc::ed25519::curve::ed25519::Ed25519; use plonky2x::ecc::ed25519::gadgets::curve::{AffinePointTarget, CircuitBuilderCurve}; use plonky2x::hash::sha::sha256::{sha256, sha256_variable_length_single_chunk}; -use plonky2x::num::u32::gadgets::arithmetic_u32::{CircuitBuilderU32, U32Target}; +use plonky2x::num::u32::gadgets::arithmetic_u32::CircuitBuilderU32; use plonky2::iop::target::Target; use crate::utils::{ EncTendermintHashTarget, I64Target, MarshalledValidatorTarget, TendermintHashTarget, - HASH_SIZE_BITS, NUM_POSSIBLE_VALIDATOR_BYTE_LENGTHS, PROTOBUF_HASH_SIZE_BITS, - VALIDATOR_BYTE_LENGTH_MAX, VALIDATOR_BYTE_LENGTH_MIN, VALIDATOR_SET_SIZE_MAX, + HASH_SIZE_BITS, PROTOBUF_HASH_SIZE_BITS, + VALIDATOR_BYTE_LENGTH_MAX, VALIDATOR_SET_SIZE_MAX, VOTING_POWER_BITS_LENGTH_MAX, VOTING_POWER_BYTES_LENGTH_MAX, VALIDATOR_BIT_LENGTH_MAX, }; @@ -475,9 +474,6 @@ impl, const D: usize> TendermintMarshaller #[cfg(test)] pub(crate) mod tests { use super::*; - use curta::math::goldilocks::cubic::GoldilocksCubicParameters; - use num::BigUint; - use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::field::types::Field; use plonky2::iop::target::BoolTarget; use plonky2::{ @@ -501,13 +497,10 @@ pub(crate) mod tests { use plonky2x::num::u32::gadgets::arithmetic_u32::U32Target; use crate::{ - signature::TendermintSignature, utils::{f_bits_to_bytes, to_be_bits}, validator::{I64Target, TendermintMarshaller}, }; - use super::VALIDATOR_BYTE_LENGTH_MIN; - type C = PoseidonGoldilocksConfig; type F = >::F; type Curve = Ed25519; @@ -543,7 +536,7 @@ pub(crate) mod tests { validator_enabled[i] = builder._true(); } - for i in validators.len()..VALIDATOR_SET_SIZE_MAX { + for _ in validators.len()..VALIDATOR_SET_SIZE_MAX { validator_byte_length.push(builder.constant(F::from_canonical_usize(0))); } return (validators_target, validator_byte_length, validator_enabled);