Add utils subcrate and fix tests

Cargo.toml

@@ -33,7 +33,14 @@ itertools = "0.9"
 lurk-macros = { path = "lurk-macros" }
 lurk-metrics = { path = "lurk-metrics" }
 metrics = { workspace = true }
-neptune = { workspace = true, features = ["arity2","arity4","arity8","arity16","pasta","bls"] }
+neptune = { workspace = true, features = [
+    "arity2",
+    "arity4",
+    "arity8",
+    "arity16",
+    "pasta",
+    "bls",
+] }
 nom = "7.1.3"
 nom_locate = "4.1.0"
 nova = { workspace = true }
@@ -57,7 +64,7 @@ serde_repr = "0.1.14"
 tap = "1.0.1"
 stable_deref_trait = "1.2.0"
 thiserror = { workspace = true }
-abomonation = { workspace = true}
+abomonation = { workspace = true }
 abomonation_derive = { git = "https://github.com/lurk-lab/abomonation_derive.git" }
 crossbeam = "0.8.2"
 byteorder = "1.4.3"
@@ -77,12 +84,17 @@ pasta-msm = { workspace = true }
 proptest = { workspace = true }
 proptest-derive = { workspace = true }
 rand = "0.8.5"
-rustyline = { version = "11.0", features = ["derive", "with-file-history"], default-features = false }
+rustyline = { version = "11.0", features = [
+    "derive",
+    "with-file-history",
+], default-features = false }
 home = "0.5.5"
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 getrandom = { version = "0.2", features = ["js"] }
-rustyline = { version = "11.0", features = ["derive"], default-features = false }
+rustyline = { version = "11.0", features = [
+    "derive",
+], default-features = false }
 
 [features]
 default = []
@@ -103,18 +115,14 @@ structopt = { version = "0.3", default-features = false }
 tap = "1.0.1"
 tempfile = { workspace = true }
 tracing-test = "0.1"
+lurk-utils = { path = "utils" }
 
 [build-dependencies]
 vergen = { version = "8", features = ["build", "git", "gitcl"] }
 
 [workspace]
 resolver = "2"
-members = [
-    "clutch",
-    "fcomm",
-    "lurk-macros",
-    "lurk-metrics"
-]
+members = ["clutch", "fcomm", "lurk-macros", "lurk-metrics", "utils"]
 
 # Dependencies that should be kept in sync through the whole workspace
 [workspace.dependencies]
@@ -187,4 +195,4 @@ harness = false
 
 [patch.crates-io]
 # This is needed to ensure halo2curves, which imports pasta-curves, uses the *same* traits in bn256_grumpkin
-pasta_curves = { git="https://github.com/lurk-lab/pasta_curves", branch="dev" }
+pasta_curves = { git = "https://github.com/lurk-lab/pasta_curves", branch = "dev" }

benches/end2end.rs

@@ -17,12 +17,11 @@ use lurk::{
     state::State,
     store::Store,
 };
+use lurk_utils::{set_public_params_dir, PUBLIC_PARAMS_BENCH_PATH};
 use pasta_curves::pallas;
 use std::time::Duration;
 use std::{cell::RefCell, rc::Rc, sync::Arc};
 
-//TODO: Define a shared bench/test function to set the PUBLIC_PARAMS env var to not pollute the disk
-//const PUBLIC_PARAMS_PATH: &str = "/var/tmp/lurk_benches/public_params";
 const DEFAULT_REDUCTION_COUNT: usize = 10;
 
 fn go_base<F: LurkField>(
@@ -68,6 +67,7 @@ fn end2end_benchmark(c: &mut Criterion) {
     let mut store = Store::default();
     let env = empty_sym_env(&store);
     let prover = NovaProver::new(reduction_count, lang_pallas);
+    set_public_params_dir(PUBLIC_PARAMS_BENCH_PATH);
 
     // use cached public params
     let pp =

benches/fibonacci.rs

@@ -20,8 +20,7 @@ use lurk::{
     state::State,
     store::Store,
 };
-
-//const PUBLIC_PARAMS_PATH: &str = "/var/tmp/lurk_benches/public_params";
+use lurk_utils::{set_public_params_dir, PUBLIC_PARAMS_BENCH_PATH};
 
 fn fib<F: LurkField>(store: &mut Store<F>, state: Rc<RefCell<State>>, _a: u64) -> Ptr<F> {
     let program = r#"
@@ -109,6 +108,7 @@ fn fibo_prove<M: measurement::Measurement>(
 fn fibonacci_prove(c: &mut Criterion) {
     tracing::debug!("{:?}", &*lurk::config::CONFIG);
     let reduction_counts = vec![100, 600, 700, 800, 900];
+    set_public_params_dir(PUBLIC_PARAMS_BENCH_PATH);
     let batch_sizes = vec![100, 200];
     let mut group: BenchmarkGroup<'_, _> = c.benchmark_group("Prove");
     group.sampling_mode(SamplingMode::Flat); // This can take a *while*

benches/public_params.rs

@@ -5,6 +5,7 @@ use lurk::{
     proof::groth16::Groth16Prover,
     proof::nova,
 };
+use lurk_utils::{set_public_params_dir, PUBLIC_PARAMS_BENCH_PATH};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -21,6 +22,7 @@ fn public_params_benchmark(c: &mut Criterion) {
     let lang_pallas =
         Lang::<pasta_curves::pallas::Scalar, Coproc<pasta_curves::pallas::Scalar>>::new();
     let lang_pallas_rc = Arc::new(lang_pallas);
+    set_public_params_dir(PUBLIC_PARAMS_BENCH_PATH);
 
     let reduction_count = DEFAULT_REDUCTION_COUNT;
 

benches/sha256_ivc.rs

@@ -9,6 +9,7 @@
 use lurk::circuit::gadgets::data::GlobalAllocations;
 use lurk::state::user_sym;
 use lurk::{circuit::gadgets::pointer::AllocatedContPtr, tag::Tag};
+use lurk_utils::{set_public_params_dir, PUBLIC_PARAMS_BENCH_PATH};
 use std::{cell::RefCell, marker::PhantomData, rc::Rc, sync::Arc, time::Duration};
 
 use bellpepper::gadgets::{multipack::pack_bits, sha256::sha256};
@@ -38,8 +39,6 @@ use lurk::{
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 
-//const PUBLIC_PARAMS_PATH: &str = "/var/tmp/lurk_benches/public_params";
-
 fn sha256_ivc<F: LurkField>(
     store: &mut Store<F>,
     state: Rc<RefCell<State>>,
@@ -261,6 +260,7 @@ fn sha256_ivc_prove<M: measurement::Measurement>(
 fn prove_benchmarks(c: &mut Criterion) {
     tracing::debug!("{:?}", &*lurk::config::CONFIG);
     let reduction_counts = vec![10, 100];
+    set_public_params_dir(PUBLIC_PARAMS_BENCH_PATH);
     let batch_sizes = vec![1, 2, 5, 10, 20];
     let mut group: BenchmarkGroup<'_, _> = c.benchmark_group("prove");
     group.sampling_mode(SamplingMode::Flat); // This can take a *while*
@@ -340,6 +340,7 @@ fn sha256_ivc_prove_compressed<M: measurement::Measurement>(
 fn prove_compressed_benchmarks(c: &mut Criterion) {
     tracing::debug!("{:?}", &*lurk::config::CONFIG);
     let reduction_counts = vec![10, 100];
+    set_public_params_dir(PUBLIC_PARAMS_BENCH_PATH);
     let batch_sizes = vec![1, 2, 5, 10, 20];
     let mut group: BenchmarkGroup<'_, _> = c.benchmark_group("prove_compressed");
     group.sampling_mode(SamplingMode::Flat); // This can take a *while*

fcomm/src/lib.rs

@@ -1183,11 +1183,6 @@ mod test {
     // Minimal chained functional commitment test
     #[test]
     fn lurk_chained_functional_commitment() {
-        // TODO: Would it be better to give a single config option that replaces the `~/.lurk` dir with a custom location,
-        // but preserves the `proofs`, `commits`, and `public_params` subdirs for simplicity?
-        // Currently the Lurk application builder must specify each settings individually if non-default locations are desired,
-        // e.g. for tests
-        // Solution: Make a pub helper function to easily set each setting similar to set_lurk_dirs, rather than changing the whole architecture
         let fcomm_path_key = "FCOMM_DATA_PATH";
         let tmp_dir = Builder::new().prefix("tmp").tempdir().expect("tmp dir");
         let tmp_dir_path = Utf8Path::from_path(tmp_dir.path()).unwrap();

src/cli/mod.rs

@@ -350,7 +350,7 @@ impl ReplCli {
             &self.public_params_dir,
             &self.proofs_dir,
             &self.commits_dir,
-            &None,
+            &self.circom_dir,
         );
         tracing::info!("Configured variables: {:?}", LURK_CACHE_CONFIG);
         let rc = get_parsed_usize("rc", &self.rc, DEFAULT_RC)?;
@@ -388,7 +388,7 @@ impl LoadCli {
             &self.public_params_dir,
             &self.proofs_dir,
             &self.commits_dir,
-            &None,
+            &self.circom_dir,
         );
         tracing::info!("Configured variables: {:?}", LURK_CACHE_CONFIG);
         let rc = get_parsed_usize("rc", &self.rc, DEFAULT_RC)?;
@@ -505,10 +505,18 @@ pub fn parse_and_run() -> Result<()> {
     }
 }
 
-#[cfg(tests)]
+#[cfg(test)]
 mod tests {
-    use crate::public_parameters::disk_cache::LURK_CACHE_CONFIG;
-
+    use camino::Utf8Path;
+    use config::{Config, Environment};
+    use std::io::prelude::*;
+    use std::{collections::HashMap, fs::File};
+    use tempfile::Builder;
+    use tracing_subscriber::{fmt, prelude::*, EnvFilter, Registry};
+    use tracing_texray::TeXRayLayer;
+
+    // Tests a generic config file with identical syntax to that used in `LURK_CACHE_CONFIG`
+    // Also tests overwriting this config file with env vars to simulate usage with CLI
     #[test]
     fn test_config_file() {
         let subscriber = Registry::default()
@@ -526,6 +534,7 @@ mod tests {
         let public_params_dir = tmp_dir.join("public_params").into_string();
         let proofs_dir = tmp_dir.join("proofs").into_string();
         let commits_dir = tmp_dir.join("commits").into_string();
+        let circom_dir = tmp_dir.join("circom").into_string();
 
         let mut config_file = File::create(&config_dir).unwrap();
         config_file
@@ -537,17 +546,23 @@ mod tests {
         config_file
             .write_all(format!("commits = \"{commits_dir}\"\n").as_bytes())
             .unwrap();
+        config_file
+            .write_all(format!("circom = \"{circom_dir}\"\n").as_bytes())
+            .unwrap();
 
-        // Overwrite proof dir with env var
+        // Overwrite proof and public param dir with env vars
         let proofs_dir_env = tmp_dir.join("proofs_env").into_string();
-
+        let public_params_dir_env = tmp_dir.join("public_params_env");
         std::env::set_var("LURK_PROOFS", proofs_dir_env.clone());
+        std::env::set_var("LURK_PUBLIC_PARAMS", public_params_dir_env.clone());
 
-        assert_eq!(
-            LURK_CACHE_CONFIG.get("public_params").unwrap(),
-            &public_params_dir
-        );
-        assert_eq!(LURK_CACHE_CONFIG.get("proofs").unwrap(), &proofs_dir_env);
-        assert_eq!(LURK_CACHE_CONFIG.get("commits").unwrap(), &commits_dir);
+        let builder = Config::builder().add_source(config::File::with_name(config_dir.as_str()));
+        let builder = builder.add_source(Environment::with_prefix("LURK"));
+        let config: HashMap<String, String> = builder.build().unwrap().try_deserialize().unwrap();
+
+        assert_eq!(config.get("public_params").unwrap(), &public_params_dir_env);
+        assert_eq!(config.get("proofs").unwrap(), &proofs_dir_env);
+        assert_eq!(config.get("commits").unwrap(), &commits_dir);
+        assert_eq!(config.get("circom").unwrap(), &circom_dir);
     }
 }

src/cli/paths.rs

@@ -74,15 +74,14 @@ fn lurk_leaf_dirs() -> [Utf8PathBuf; 4] {
     ]
 }
 
-/// Propagates the CLI arg overrides by setting env vars before `LURK_CACHE_DIR` is initialized.
+/// Propagates the CLI arg overrides by setting env vars before `LURK_CACHE_CONFIG` is initialized.
 /// If called afterwards it has no effect, as these env vars aren't read by anything else.
 pub(crate) fn set_lurk_dirs(
     public_params_dir: &Option<Utf8PathBuf>,
     proofs_dir: &Option<Utf8PathBuf>,
     commits_dir: &Option<Utf8PathBuf>,
     circom_dir: &Option<Utf8PathBuf>,
 ) {
-    // TODO: Test the env var name is correct
     let set_path = |cli_path_override: &Option<Utf8PathBuf>, env_key: &str| {
         if let Some(path) = cli_path_override {
             std::env::set_var(env_key, path);
@@ -94,7 +93,7 @@ pub(crate) fn set_lurk_dirs(
     set_path(commits_dir, "LURK_COMMITS");
     set_path(circom_dir, "LURK_CIRCOM");
 
-    // Initializes `LURK_CACHE_DIR` and creates dirs for each of the above settings
+    // Initializes `LURK_CACHE_CONFIG` and creates dirs for each of the above settings
     create_lurk_dirs().unwrap();
 }
 

src/public_parameters/disk_cache.rs

@@ -13,7 +13,6 @@ use crate::coprocessor::Coprocessor;
 use crate::proof::nova::{CurveCycleEquipped, PublicParams, G1, G2};
 use crate::public_parameters::error::Error;
 
-// TODO: Integrate or disambiguate with other config settings like `src/config.rs`
 /// Contains the Lurk config settings
 /// The config file is located in `$HOME/.config/lurk.toml` or `<current_dir>/.config/lurk.toml` on WASM.
 /// The config syntax is as follows:
@@ -24,7 +23,7 @@ use crate::public_parameters::error::Error;
 /// ```sh
 /// LURK_PUBLIC_PARAMS="/path/to/params"
 /// ```
-pub(crate) static LURK_CACHE_CONFIG: Lazy<HashMap<&str, String>> = Lazy::new(|| {
+pub(crate) static LURK_CACHE_CONFIG: Lazy<HashMap<String, String>> = Lazy::new(|| {
     tracing::info!("Initializing Lurk cache config");
     // First load from the config file if it exists
     let mut config_path = Utf8PathBuf::new();

utils/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "lurk-utils"
+version = "0.1.0"
+license = "MIT OR Apache-2.0"
+description = "Lurk utility functions"
+edition = "2021"
+rust-version = "1.70.0"

utils/src/lib.rs

@@ -0,0 +1,7 @@
+pub const PUBLIC_PARAMS_BENCH_PATH: &str = "/var/tmp/lurk_benches/public_params";
+// Note: Must be called before `LURK_CACHE_CONFIG` is initialized,
+// e.g. by calling `public_params()`, in order to override
+// the preconfigured or default location
+pub fn set_public_params_dir(path: &str) {
+    std::env::set_var("LURK_PUBLIC_PARAMS", path);
+}