add max-chunk-size as a REPL parameter for memory control

benches/end2end.rs

@@ -84,8 +84,8 @@ fn end2end_benchmark(c: &mut Criterion) {
     group.bench_with_input(benchmark_id, &size, |b, &s| {
         b.iter(|| {
             let ptr = go_base::<Fq>(&store, state.clone(), s.0, s.1);
-            let (frames, _) = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
-            let _result = prover.prove(&pp, &frames, &store).unwrap();
+            let frames = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
+            let _result = prover.prove(&pp, &frames, &store, None).unwrap();
         })
     });
 
@@ -251,10 +251,10 @@ fn prove_benchmark(c: &mut Criterion) {
         let ptr = go_base::<Fq>(&store, state.clone(), s.0, s.1);
         let prover: NovaProver<'_, Fq, Coproc<Fq>, MultiFrame<'_, Fq, Coproc<Fq>>> =
             NovaProver::new(reduction_count, lang_pallas_rc.clone());
-        let (frames, _) = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
+        let frames = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
 
         b.iter(|| {
-            let result = prover.prove(&pp, &frames, &store).unwrap();
+            let result = prover.prove(&pp, &frames, &store, None).unwrap();
             black_box(result);
         })
     });
@@ -298,10 +298,10 @@ fn prove_compressed_benchmark(c: &mut Criterion) {
     group.bench_with_input(benchmark_id, &size, |b, &s| {
         let ptr = go_base::<Fq>(&store, state.clone(), s.0, s.1);
         let prover = NovaProver::new(reduction_count, lang_pallas_rc.clone());
-        let (frames, _) = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
+        let frames = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
 
         b.iter(|| {
-            let (proof, _, _, _) = prover.prove(&pp, &frames, &store).unwrap();
+            let (proof, _, _, _) = prover.prove(&pp, &frames, &store, None).unwrap();
 
             let compressed_result = proof.compress(&pp).unwrap();
             black_box(compressed_result);
@@ -344,8 +344,8 @@ fn verify_benchmark(c: &mut Criterion) {
         group.bench_with_input(benchmark_id, &size, |b, &s| {
             let ptr = go_base(&store, state.clone(), s.0, s.1);
             let prover = NovaProver::new(reduction_count, lang_pallas_rc.clone());
-            let (frames, _) = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
-            let (proof, z0, zi, num_steps) = prover.prove(&pp, &frames, &store).unwrap();
+            let frames = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
+            let (proof, z0, zi, num_steps) = prover.prove(&pp, &frames, &store, None).unwrap();
 
             b.iter_batched(
                 || z0.clone(),
@@ -396,8 +396,8 @@ fn verify_compressed_benchmark(c: &mut Criterion) {
         group.bench_with_input(benchmark_id, &size, |b, &s| {
             let ptr = go_base(&store, state.clone(), s.0, s.1);
             let prover = NovaProver::new(reduction_count, lang_pallas_rc.clone());
-            let (frames, _) = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
-            let (proof, z0, zi, num_steps) = prover.prove(&pp, &frames, &store).unwrap();
+            let frames = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
+            let (proof, z0, zi, num_steps) = prover.prove(&pp, &frames, &store, None).unwrap();
 
             let compressed_proof = proof.compress(&pp).unwrap();
 

benches/fibonacci.rs

@@ -142,13 +142,12 @@ fn fibonacci_prove<M: measurement::Measurement>(
 
             let frames =
                 &evaluate::<pasta_curves::Fq, Coproc<pasta_curves::Fq>>(None, ptr, &store, limit)
-                    .unwrap()
-                    .0;
+                    .unwrap();
 
             b.iter_batched(
                 || frames,
                 |frames| {
-                    let result = prover.prove(&pp, frames, &store);
+                    let result = prover.prove(&pp, frames, &store, None);
                     let _ = black_box(result);
                 },
                 BatchSize::LargeInput,

benches/sha256.rs

@@ -134,14 +134,12 @@ fn sha256_ivc_prove<M: measurement::Measurement>(
 
             let prover = NovaProver::new(prove_params.reduction_count, lang_rc.clone());
 
-            let frames = &evaluate(Some((&lurk_step, &lang)), ptr, store, limit)
-                .unwrap()
-                .0;
+            let frames = &evaluate(Some((&lurk_step, &lang)), ptr, store, limit).unwrap();
 
             b.iter_batched(
                 || frames,
                 |frames| {
-                    let result = prover.prove(&pp, frames, store);
+                    let result = prover.prove(&pp, frames, store, None);
                     let _ = black_box(result);
                 },
                 BatchSize::LargeInput,
@@ -217,14 +215,12 @@ fn sha256_ivc_prove_compressed<M: measurement::Measurement>(
 
             let prover = NovaProver::new(prove_params.reduction_count, lang_rc.clone());
 
-            let frames = &evaluate(Some((&lurk_step, &lang)), ptr, store, limit)
-                .unwrap()
-                .0;
+            let frames = &evaluate(Some((&lurk_step, &lang)), ptr, store, limit).unwrap();
 
             b.iter_batched(
                 || frames,
                 |frames| {
-                    let (proof, _, _, _) = prover.prove(&pp, frames, store).unwrap();
+                    let (proof, _, _, _) = prover.prove(&pp, frames, store, None).unwrap();
                     let compressed_result = proof.compress(&pp).unwrap();
 
                     let _ = black_box(compressed_result);
@@ -302,14 +298,12 @@ fn sha256_nivc_prove<M: measurement::Measurement>(
 
             let prover = SuperNovaProver::new(prove_params.reduction_count, lang_rc.clone());
 
-            let frames = &evaluate(Some((&lurk_step, &lang)), ptr, store, limit)
-                .unwrap()
-                .0;
+            let frames = &evaluate(Some((&lurk_step, &lang)), ptr, store, limit).unwrap();
 
             b.iter_batched(
                 || frames,
                 |frames| {
-                    let result = prover.prove(&pp, frames, store);
+                    let result = prover.prove(&pp, frames, store, None);
                     let _ = black_box(result);
                 },
                 BatchSize::LargeInput,

benches/synthesis.rs

@@ -50,7 +50,7 @@ fn synthesize<M: measurement::Measurement>(
             let store = Store::default();
             let fib_n = (reduction_count / 3) as u64; // Heuristic, since one fib is 35 iterations.
             let ptr = fib::<pasta_curves::Fq>(&store, state.clone(), black_box(fib_n));
-            let (frames, _) = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
+            let frames = evaluate::<Fq, Coproc<Fq>>(None, ptr, &store, limit).unwrap();
 
             let folding_config =
                 Arc::new(FoldingConfig::new_ivc(lang_rc.clone(), *reduction_count));

examples/sha256_nivc.rs

@@ -1,5 +1,5 @@
 use pasta_curves::pallas::Scalar as Fr;
-use std::{sync::Arc, time::Instant};
+use std::{marker::PhantomData, sync::Arc, time::Instant};
 use tracing_subscriber::{fmt, prelude::*, EnvFilter, Registry};
 use tracing_texray::TeXRayLayer;
 
@@ -77,7 +77,7 @@ fn main() {
     let lang_rc = Arc::new(lang.clone());
 
     let lurk_step = make_eval_step_from_config(&EvalConfig::new_nivc(&lang));
-    let (frames, _) = evaluate(Some((&lurk_step, &lang)), call, store, 1000).unwrap();
+    let frames = evaluate(Some((&lurk_step, &lang)), call, store, 1000).unwrap();
 
     let supernova_prover = SuperNovaProver::<Fr, Sha256Coproc<Fr>, MultiFrame<'_, _, _>>::new(
         REDUCTION_COUNT,
@@ -95,17 +95,16 @@ fn main() {
 
     println!("Beginning proof step...");
     let proof_start = Instant::now();
-    let ((proof, last_circuit_index), z0, zi, _num_steps) =
-        tracing_texray::examine(tracing::info_span!("bang!"))
-            .in_scope(|| supernova_prover.prove(&pp, &frames, store).unwrap());
+    let (proof, z0, zi, _num_steps) = tracing_texray::examine(tracing::info_span!("bang!"))
+        .in_scope(|| supernova_prover.prove(&pp, &frames, store, None).unwrap());
     let proof_end = proof_start.elapsed();
 
     println!("Proofs took {:?}", proof_end);
 
     println!("Verifying proof...");
 
     let verify_start = Instant::now();
-    assert!(proof.verify(&pp, &z0, &zi, last_circuit_index).unwrap());
+    assert!(proof.verify(&pp, &z0, &zi, PhantomData).unwrap());
     let verify_end = verify_start.elapsed();
 
     println!("Verify took {:?}", verify_end);
@@ -118,9 +117,7 @@ fn main() {
     println!("Compression took {:?}", compress_end);
 
     let compressed_verify_start = Instant::now();
-    let res = compressed_proof
-        .verify(&pp, &z0, &zi, last_circuit_index)
-        .unwrap();
+    let res = compressed_proof.verify(&pp, &z0, &zi, PhantomData).unwrap();
     let compressed_verify_end = compressed_verify_start.elapsed();
 
     println!("Final verification took {:?}", compressed_verify_end);

src/cli/config.rs

@@ -60,6 +60,8 @@ pub(crate) struct CliSettings {
     /// Iteration limit for the program, which is arbitrary to user preferences
     /// Used mainly as a safety check, similar to default stack size
     pub(crate) limit: usize,
+    /// Maximum number of frames held at once, used to avoid memory overflows
+    pub(crate) max_chunk_size: usize,
 }
 
 impl CliSettings {
@@ -68,14 +70,15 @@ impl CliSettings {
         config_file: &Utf8PathBuf,
         cli_settings: Option<&HashMap<&str, String>>,
     ) -> Result<Self, ConfigError> {
-        let (proofs, commits, circom, backend, field, rc, limit) = (
+        let (proofs, commits, circom, backend, field, rc, limit, max_chunk_size) = (
             "proofs_dir",
             "commits_dir",
             "circom_dir",
             "backend",
             "field",
             "rc",
             "limit",
+            "max_chunk_size",
         );
         Config::builder()
             .set_default(proofs, proofs_default_dir().to_string())?
@@ -96,6 +99,7 @@ impl CliSettings {
             .set_override_option(field, cli_settings.and_then(|s| s.get(field).map(|v| v.to_owned())))?
             .set_override_option(rc, cli_settings.and_then(|s| s.get(rc).map(|v| v.to_owned())))?
             .set_override_option(limit, cli_settings.and_then(|s| s.get(limit).map(|v| v.to_owned())))?
+            .set_override_option(max_chunk_size, cli_settings.and_then(|s| s.get(max_chunk_size).map(|v| v.to_owned())))?
             .build()
             .and_then(|c| c.try_deserialize())
     }
@@ -111,6 +115,7 @@ impl Default for CliSettings {
             field: LanguageField::default(),
             rc: 10,
             limit: 100_000_000,
+            max_chunk_size: 1_000_000,
         }
     }
 }
@@ -140,6 +145,7 @@ mod tests {
         let field = "Pallas";
         let rc = 100;
         let limit = 100_000;
+        let max_chunk_size = 10_000;
 
         let mut config_file = std::fs::File::create(config_dir.clone()).unwrap();
         config_file
@@ -166,6 +172,9 @@ mod tests {
         config_file
             .write_all(format!("limit = {limit}\n").as_bytes())
             .unwrap();
+        config_file
+            .write_all(format!("max_chunk_size = {max_chunk_size}\n").as_bytes())
+            .unwrap();
 
         let cli_config = CliSettings::from_config(&config_dir, None).unwrap();
         let lurk_config = Settings::from_config(&config_dir, None).unwrap();
@@ -177,5 +186,6 @@ mod tests {
         assert_eq!(cli_config.field, LanguageField::Pallas);
         assert_eq!(cli_config.rc, rc);
         assert_eq!(cli_config.limit, limit);
+        assert_eq!(cli_config.max_chunk_size, max_chunk_size);
     }
 }

src/cli/mod.rs

@@ -89,6 +89,10 @@ struct LoadArgs {
     #[clap(long, value_parser)]
     limit: Option<usize>,
 
+    /// Maximum number of held frames (defaults to 1_000_000; rounded up to the next multiple of rc)
+    #[clap(long, value_parser)]
+    max_chunk_size: Option<usize>,
+
     /// Prover backend (defaults to "Nova")
     #[clap(long, value_enum)]
     backend: Option<Backend>,
@@ -138,6 +142,9 @@ struct LoadCli {
     #[clap(long, value_parser)]
     limit: Option<usize>,
 
+    #[clap(long, value_parser)]
+    max_chunk_size: Option<usize>,
+
     #[clap(long, value_enum)]
     backend: Option<Backend>,
 
@@ -169,6 +176,7 @@ impl LoadArgs {
             config: self.config,
             rc: self.rc,
             limit: self.limit,
+            max_chunk_size: self.max_chunk_size,
             backend: self.backend,
             field: self.field,
             public_params_dir: self.public_params_dir,
@@ -202,6 +210,10 @@ struct ReplArgs {
     #[clap(long, value_parser)]
     limit: Option<usize>,
 
+    /// Maximum number of held frames (defaults to 1_000_000; rounded up to the next multiple of rc)
+    #[clap(long, value_parser)]
+    max_chunk_size: Option<usize>,
+
     /// Prover backend (defaults to "Nova")
     #[clap(long, value_enum)]
     backend: Option<Backend>,
@@ -244,6 +256,9 @@ struct ReplCli {
     #[clap(long, value_parser)]
     limit: Option<usize>,
 
+    #[clap(long, value_parser)]
+    max_chunk_size: Option<usize>,
+
     #[clap(long, value_enum)]
     backend: Option<Backend>,
 
@@ -271,6 +286,7 @@ impl ReplArgs {
             config: self.config,
             rc: self.rc,
             limit: self.limit,
+            max_chunk_size: self.max_chunk_size,
             backend: self.backend,
             field: self.field,
             public_params_dir: self.public_params_dir,
@@ -302,17 +318,17 @@ fn get_store<F: LurkField + for<'a> serde::de::Deserialize<'a>>(
 }
 
 macro_rules! new_repl {
-    ( $cli: expr, $rc: expr, $limit: expr, $field: path, $backend: expr ) => {{
+    ( $cli: expr, $rc: expr, $limit: expr, $max_chunk_size: expr, $field: path, $backend: expr ) => {{
         let store = get_store(&$cli.zstore).with_context(|| "reading store from file")?;
-        Repl::<$field>::new(store, $rc, $limit, $backend)
+        Repl::<$field>::new(store, $rc, $limit, $max_chunk_size, $backend)
     }};
 }
 
 impl ReplCli {
     fn run(&self) -> Result<()> {
         macro_rules! repl {
-            ( $rc: expr, $limit: expr, $field: path, $backend: expr ) => {{
-                let mut repl = new_repl!(self, $rc, $limit, $field, $backend);
+            ( $rc: expr, $limit: expr, $max_chunk_size: expr, $field: path, $backend: expr ) => {{
+                let mut repl = new_repl!(self, $rc, $limit, $max_chunk_size, $field, $backend);
                 if let Some(lurk_file) = &self.load {
                     repl.load_file(lurk_file, false)?;
                 }
@@ -338,7 +354,8 @@ impl ReplCli {
             backend,
             field,
             rc,
-            limit
+            limit,
+            max_chunk_size
         );
 
         // Initializes CLI config with CLI arguments as overrides
@@ -348,12 +365,15 @@ impl ReplCli {
 
         let rc = config.rc;
         let limit = config.limit;
+        let max_chunk_size = config.max_chunk_size;
         let backend = &config.backend;
         let field = &config.field;
         validate_non_zero("rc", rc)?;
         backend.validate_field(field)?;
         match field {
-            LanguageField::Pallas => repl!(rc, limit, pallas::Scalar, backend.clone()),
+            LanguageField::Pallas => {
+                repl!(rc, limit, max_chunk_size, pallas::Scalar, backend.clone())
+            }
             LanguageField::Vesta => todo!(),
             LanguageField::BN256 => todo!(),
             LanguageField::Grumpkin => todo!(),
@@ -364,11 +384,11 @@ impl ReplCli {
 impl LoadCli {
     fn run(&self) -> Result<()> {
         macro_rules! load {
-            ( $rc: expr, $limit: expr, $field: path, $backend: expr ) => {{
-                let mut repl = new_repl!(self, $rc, $limit, $field, $backend);
+            ( $rc: expr, $limit: expr, $max_chunk_size: expr, $field: path, $backend: expr ) => {{
+                let mut repl = new_repl!(self, $rc, $limit, $max_chunk_size, $field, $backend);
                 repl.load_file(&self.lurk_file, self.demo)?;
                 if self.prove {
-                    repl.prove_last_frames()?;
+                    repl.prove_last_computation()?;
                 }
                 Ok(())
             }};
@@ -392,7 +412,8 @@ impl LoadCli {
             backend,
             field,
             rc,
-            limit
+            limit,
+            max_chunk_size
         );
 
         // Initializes CLI config with CLI arguments as overrides
@@ -402,12 +423,15 @@ impl LoadCli {
 
         let rc = config.rc;
         let limit = config.limit;
+        let max_chunk_size = config.max_chunk_size;
         let backend = &config.backend;
         let field = &config.field;
         validate_non_zero("rc", rc)?;
         backend.validate_field(field)?;
         match field {
-            LanguageField::Pallas => load!(rc, limit, pallas::Scalar, backend.clone()),
+            LanguageField::Pallas => {
+                load!(rc, limit, max_chunk_size, pallas::Scalar, backend.clone())
+            }
             LanguageField::Vesta => todo!(),
             LanguageField::BN256 => todo!(),
             LanguageField::Grumpkin => todo!(),