Merge pull request #219 from a16z/feat/serialize-proofs

feat: Serialize proofs

common/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.2.0"
 edition = "2021"
 
 [dependencies]
+ark-serialize = { version = "0.4.2", features = ["derive"] }
 serde = { version = "1.0.193", features = ["derive"] }
 serde_json = "1.0.108"
 strum_macros = "0.25.3"

common/src/rv_trace.rs

@@ -1,4 +1,5 @@
 use crate::constants::{MEMORY_OPS_PER_INSTRUCTION, PANIC_ADDRESS, INPUT_START_ADDRESS, OUTPUT_START_ADDRESS};
+use ark_serialize::{CanonicalDeserialize, CanonicalSerialize};
 use serde::{Deserialize, Serialize};
 use strum_macros::FromRepr;
 
@@ -611,7 +612,7 @@ impl RV32IM {
 /// all reads from the reserved memory address space for program inputs and all writes
 /// to the reserved memory address space for program outputs.
 /// The inputs and outputs are part of the public inputs to the proof.
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, CanonicalSerialize, CanonicalDeserialize)]
 pub struct JoltDevice {
     pub inputs: Vec<u8>,
     pub outputs: Vec<u8>,

jolt-core/src/benches/bench.rs

@@ -6,6 +6,7 @@ use crate::jolt::vm::rv32i_vm::{RV32IJoltVM, C, M, RV32I};
 use crate::jolt::vm::Jolt;
 use crate::poly::dense_mlpoly::bench::{init_commit_bench, run_commit_bench};
 use ark_bn254::{Fr, G1Projective};
+use ark_serialize::CanonicalSerialize;
 use common::rv_trace::{ELFInstruction, JoltDevice};
 use criterion::black_box;
 use merlin::Transcript;
@@ -220,6 +221,16 @@ fn sha3() -> Vec<(tracing::Span, Box<dyn FnOnce()>)> {
     prove_example("sha3-guest", &vec![5u8; 2048])
 }
 
+fn serialize_and_print_size(name: &str, item: &impl ark_serialize::CanonicalSerialize) {
+    use std::fs::File;
+    let mut file = File::create("temp_file").unwrap();
+    item.serialize_compressed(&mut file).unwrap();
+    let file_size_bytes = file.metadata().unwrap().len();
+    let file_size_kb = file_size_bytes as f64 / 1024.0;
+    let file_size_mb = file_size_kb / 1024.0;
+    println!("{:<30} : {:.3} MB", name, file_size_mb);
+}
+
 fn prove_example<T: Serialize>(
     example_name: &str,
     input: &T,
@@ -246,6 +257,15 @@ fn prove_example<T: Serialize>(
             circuit_flags,
             preprocessing.clone(),
         );
+
+        println!("Proof sizing:");
+        serialize_and_print_size("jolt_commitments", &jolt_commitments);
+        serialize_and_print_size("jolt_proof", &jolt_proof);
+        serialize_and_print_size(" jolt_proof.r1cs", &jolt_proof.r1cs);
+        serialize_and_print_size(" jolt_proof.bytecode", &jolt_proof.bytecode);
+        serialize_and_print_size(" jolt_proof.read_write_memory", &jolt_proof.read_write_memory);
+        serialize_and_print_size(" jolt_proof.instruction_lookups", &jolt_proof.instruction_lookups);
+
         let verification_result = RV32IJoltVM::verify(preprocessing, jolt_proof, jolt_commitments);
         assert!(
             verification_result.is_ok(),

jolt-core/src/jolt/instruction/mod.rs

@@ -12,7 +12,7 @@ use common::rv_trace::ELFInstruction;
 use std::fmt::Debug;
 
 #[enum_dispatch]
-pub trait JoltInstruction: Sync + Clone + Debug {
+pub trait JoltInstruction: Clone + Debug + Send + Sync {
     fn operands(&self) -> [u64; 2];
     /// Combines `vals` according to the instruction's "collation" polynomial `g`.
     /// If `vals` are subtable entries (as opposed to MLE evaluations), this function returns the
@@ -70,7 +70,7 @@ pub trait JoltInstruction: Sync + Clone + Debug {
 }
 
 pub trait JoltInstructionSet:
-    JoltInstruction + IntoEnumIterator + EnumCount + for<'a> TryFrom<&'a ELFInstruction>
+    JoltInstruction + IntoEnumIterator + EnumCount + for<'a> TryFrom<&'a ELFInstruction> + Send + Sync
 {
     fn enum_index(instruction: &Self) -> usize {
         unsafe { *<*const _>::from(instruction).cast::<u8>() as usize }

jolt-core/src/jolt/subtable/mod.rs

@@ -22,7 +22,7 @@ pub trait LassoSubtable<F: PrimeField>: 'static + Sync {
 
 pub type SubtableId = TypeId;
 pub trait JoltSubtableSet<F: PrimeField>:
-    LassoSubtable<F> + IntoEnumIterator + EnumCount + From<SubtableId> + Into<usize>
+    LassoSubtable<F> + IntoEnumIterator + EnumCount + From<SubtableId> + Into<usize> + Send + Sync
 {
     fn enum_index(subtable: Box<dyn LassoSubtable<F>>) -> usize {
         Self::from(subtable.subtable_id()).into()

jolt-core/src/jolt/vm/bytecode.rs

@@ -15,6 +15,7 @@ use crate::poly::pedersen::PedersenGenerators;
 use common::constants::{BYTES_PER_INSTRUCTION, NUM_R1CS_POLYS, RAM_START_ADDRESS, REGISTER_COUNT};
 use common::rv_trace::ELFInstruction;
 use common::to_ram_address;
+use ark_serialize::{CanonicalSerialize, CanonicalDeserialize};
 
 use rayon::prelude::*;
 
@@ -325,6 +326,8 @@ impl<F: PrimeField, G: CurveGroup<ScalarField = F>> BytecodePolynomials<F, G> {
     }
 }
 
+
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct BytecodeCommitment<G: CurveGroup> {
     pub read_write_generators: HyraxGenerators<NUM_R1CS_POLYS, G>,
     pub read_write_commitments: Vec<HyraxCommitment<NUM_R1CS_POLYS, G>>,
@@ -569,6 +572,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct BytecodeReadWriteOpenings<F>
 where
     F: PrimeField,
@@ -657,6 +661,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct BytecodeInitFinalOpenings<F>
 where
     F: PrimeField,

jolt-core/src/jolt/vm/instruction_lookups.rs

@@ -89,6 +89,7 @@ pub struct BatchedInstructionPolynomials<F: PrimeField> {
 }
 
 /// Commitments to BatchedInstructionPolynomials.
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct InstructionCommitment<G: CurveGroup> {
     pub read_write_generators: HyraxGenerators<NUM_R1CS_POLYS, G>,
     /// Commitments to dim_i and read_cts_i polynomials.
@@ -220,6 +221,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct InstructionReadWriteOpenings<F>
 where
     F: PrimeField,
@@ -336,6 +338,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct InstructionFinalOpenings<F, Subtables>
 where
     F: PrimeField,
@@ -750,6 +753,7 @@ where
 }
 
 /// Proof of instruction lookups for a single Jolt program execution.
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct InstructionLookupsProof<const C: usize, const M: usize, F, G, InstructionSet, Subtables>
 where
     F: PrimeField,
@@ -758,10 +762,7 @@ where
     InstructionSet: JoltInstructionSet,
 {
     _instructions: PhantomData<InstructionSet>,
-    /// "Primary" sumcheck, i.e. proving \sum_x \tilde{eq}(r, x) * \sum_i flag_i(x) * g_i(E_1(x), ..., E_\alpha(x))
     primary_sumcheck: PrimarySumcheck<F, G>,
-
-    /// Memory checking proof, showing that E_i polynomials are well-formed.
     memory_checking: MemoryCheckingProof<
         G,
         InstructionPolynomials<F, G>,
@@ -770,6 +771,7 @@ where
     >,
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct PrimarySumcheck<F: PrimeField, G: CurveGroup<ScalarField = F>> {
     sumcheck_proof: SumcheckInstanceProof<F>,
     num_rounds: usize,

jolt-core/src/jolt/vm/mod.rs

@@ -1,5 +1,6 @@
 use ark_ec::CurveGroup;
 use ark_ff::PrimeField;
+use ark_serialize::{CanonicalDeserialize, CanonicalSerialize};
 use ark_std::log2;
 use common::constants::NUM_R1CS_POLYS;
 use common::rv_trace::{JoltDevice, NUM_CIRCUIT_FLAGS};
@@ -49,18 +50,19 @@ where
     pub read_write_memory: ReadWriteMemoryPreprocessing,
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct JoltProof<const C: usize, const M: usize, F, G, InstructionSet, Subtables>
 where
     F: PrimeField,
     G: CurveGroup<ScalarField = F>,
     InstructionSet: JoltInstructionSet,
     Subtables: JoltSubtableSet<F>,
 {
-    program_io: JoltDevice,
-    bytecode: BytecodeProof<F, G>,
-    read_write_memory: ReadWriteMemoryProof<F, G>,
-    instruction_lookups: InstructionLookupsProof<C, M, F, G, InstructionSet, Subtables>,
-    r1cs: R1CSProof<F, G>,
+    pub program_io: JoltDevice,
+    pub bytecode: BytecodeProof<F, G>,
+    pub read_write_memory: ReadWriteMemoryProof<F, G>,
+    pub instruction_lookups: InstructionLookupsProof<C, M, F, G, InstructionSet, Subtables>,
+    pub r1cs: R1CSProof<F, G>,
 }
 
 pub struct JoltPolynomials<F, G>
@@ -73,6 +75,7 @@ where
     pub instruction_lookups: InstructionPolynomials<F, G>,
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct JoltCommitments<G: CurveGroup> {
     pub bytecode: BytecodeCommitment<G>,
     pub read_write_memory: MemoryCommitment<G>,
@@ -152,6 +155,7 @@ pub trait Jolt<F: PrimeField, G: CurveGroup<ScalarField = F>, const C: usize, co
         JoltProof<C, M, F, G, Self::InstructionSet, Self::Subtables>,
         JoltCommitments<G>,
     ) {
+        println!("Jolt::prove({})", memory_trace.len());
         let mut transcript = Transcript::new(b"Jolt transcript");
         let (bytecode_proof, bytecode_polynomials, bytecode_commitment) = Self::prove_bytecode(
             &preprocessing.bytecode,

jolt-core/src/jolt/vm/read_write_memory.rs

@@ -38,6 +38,7 @@ use common::constants::{
 };
 use common::rv_trace::{ELFInstruction, JoltDevice, MemoryOp, RV32IM};
 use common::to_ram_address;
+use ark_serialize::{CanonicalSerialize, CanonicalDeserialize};
 
 use super::timestamp_range_check::TimestampValidityProof;
 
@@ -437,6 +438,7 @@ pub struct BatchedMemoryPolynomials<F: PrimeField> {
     pub(crate) batched_t_read_write: DensePolynomial<F>,
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct MemoryCommitment<G: CurveGroup> {
     /// Generators for a_read_write, v_read, v_write
     pub read_write_generators: HyraxGenerators<NUM_R1CS_POLYS, G>,
@@ -511,6 +513,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct MemoryReadWriteOpenings<F, G>
 where
     F: PrimeField,
@@ -528,6 +531,7 @@ where
     pub t_write_opening: [F; MEMORY_OPS_PER_INSTRUCTION],
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct MemoryReadWriteOpeningProof<G: CurveGroup> {
     a_v_opening_proof: BatchedHyraxOpeningProof<NUM_R1CS_POLYS, G>,
     t_opening_proof: ConcatenatedPolynomialOpeningProof<G>,
@@ -646,6 +650,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct MemoryInitFinalOpenings<F>
 where
     F: PrimeField,
@@ -660,6 +665,7 @@ where
     t_final: F,
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct MemoryInitFinalOpeningProof<F, G>
 where
     F: PrimeField,
@@ -949,6 +955,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct OutputSumcheckProof<F, G>
 where
     F: PrimeField,
@@ -1109,6 +1116,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct ReadWriteMemoryProof<F, G>
 where
     F: PrimeField,

jolt-core/src/jolt/vm/timestamp_range_check.rs

@@ -1,5 +1,6 @@
 use ark_ec::CurveGroup;
 use ark_ff::PrimeField;
+use ark_serialize::{CanonicalDeserialize, CanonicalSerialize};
 use common::constants::MEMORY_OPS_PER_INSTRUCTION;
 use itertools::interleave;
 use merlin::Transcript;
@@ -201,6 +202,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct RangeCheckOpenings<F, G>
 where
     F: PrimeField,
@@ -215,6 +217,7 @@ where
     identity_poly_opening: Option<F>,
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct RangeCheckOpeningProof<G>
 where
     G: CurveGroup,
@@ -589,6 +592,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct TimestampValidityProof<F, G>
 where
     F: PrimeField,

jolt-core/src/lasso/memory_checking.rs

@@ -13,12 +13,14 @@ use crate::utils::transcript::ProofTranscript;
 
 use ark_ec::CurveGroup;
 use ark_ff::PrimeField;
+use ark_serialize::{CanonicalDeserialize, CanonicalSerialize};
 use itertools::interleave;
 use merlin::Transcript;
 use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
 use std::iter::zip;
 use std::marker::PhantomData;
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct MultisetHashes<F: PrimeField> {
     /// Multiset hash of "read" tuples
     pub read_hashes: Vec<F>,
@@ -58,10 +60,11 @@ impl<F: PrimeField> MultisetHashes<F> {
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct MemoryCheckingProof<G, Polynomials, ReadWriteOpenings, InitFinalOpenings>
 where
     G: CurveGroup,
-    Polynomials: BatchablePolynomials<G> + ?Sized,
+    Polynomials: BatchablePolynomials<G>,
     ReadWriteOpenings: StructuredOpeningProof<G::ScalarField, G, Polynomials>,
     InitFinalOpenings: StructuredOpeningProof<G::ScalarField, G, Polynomials>,
 {

jolt-core/src/lasso/surge.rs

@@ -1,5 +1,6 @@
 use ark_ec::CurveGroup;
 use ark_ff::PrimeField;
+use ark_serialize::{CanonicalDeserialize, CanonicalSerialize};
 use merlin::Transcript;
 use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
 use std::marker::{PhantomData, Sync};
@@ -40,6 +41,7 @@ pub struct BatchedSurgePolynomials<F: PrimeField> {
     pub batched_E: DensePolynomial<F>,
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct SurgeCommitment<G: CurveGroup> {
     pub dim_read_commitment: ConcatenatedPolynomialCommitment<G>,
     pub final_commitment: ConcatenatedPolynomialCommitment<G>,
@@ -137,6 +139,7 @@ impl<F: PrimeField, G: CurveGroup<ScalarField = F>> StructuredOpeningProof<F, G,
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct SurgeReadWriteOpenings<F>
 where
     F: PrimeField,
@@ -146,6 +149,7 @@ where
     E_poly_openings: Vec<F>, // NUM_MEMORIES-sized
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct SurgeReadWriteOpeningProof<F, G>
 where
     F: PrimeField,
@@ -238,6 +242,7 @@ where
     }
 }
 
+#[derive(CanonicalSerialize, CanonicalDeserialize)]
 pub struct SurgeFinalOpenings<F, Instruction, const C: usize, const M: usize>
 where
     F: PrimeField,