feat: Big integer abstraction for Name accumulators & rug-based BigInt backend

wnfs-bench/Cargo.toml

@@ -16,7 +16,7 @@ rand = "0.8"
 wnfs = { path = "../wnfs" }
 wnfs-common = { path = "../wnfs-common", features = ["test_utils"] }
 wnfs-hamt = { path = "../wnfs-hamt", features = ["test_utils"] }
-wnfs-nameaccumulator = { path = "../wnfs-nameaccumulator" }
+wnfs-nameaccumulator = { path = "../wnfs-nameaccumulator", features = ["num-bigint-dig", "rug"] }
 
 [[bench]]
 name = "hamt"

wnfs-bench/nameaccumulator.rs

@@ -1,41 +1,72 @@
 use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
 use rand::{thread_rng, Rng};
-use wnfs_nameaccumulator::{AccumulatorSetup, NameAccumulator, NameSegment};
+use wnfs_nameaccumulator::{AccumulatorSetup, BigNumDig, BigNumRug, NameAccumulator, NameSegment};
 
 fn name_segment_from_digest(c: &mut Criterion) {
-    c.bench_function("NameSegment::new_hashed", |b| {
+    c.bench_function("NameSegment::<BigNumDig>::new_hashed", |b| {
         b.iter_batched(
             || thread_rng().gen::<[u8; 32]>(),
-            |sth| NameSegment::new_hashed("wnfs benchmarks", sth),
+            |sth| NameSegment::<BigNumDig>::new_hashed("wnfs benchmarks", sth),
+            BatchSize::SmallInput,
+        );
+    });
+    c.bench_function("NameSegment::<BigNumRug>::new_hashed", |b| {
+        b.iter_batched(
+            || thread_rng().gen::<[u8; 32]>(),
+            |sth| NameSegment::<BigNumRug>::new_hashed("wnfs benchmarks", sth),
             BatchSize::SmallInput,
         );
     });
 }
 
 fn name_segment_rng(c: &mut Criterion) {
-    c.bench_function("NameSegment::new(rng)", |b| {
-        b.iter(|| NameSegment::new(&mut thread_rng()));
+    c.bench_function("NameSegment::<BigNumDig>::new(rng)", |b| {
+        b.iter(|| NameSegment::<BigNumDig>::new(&mut thread_rng()));
+    });
+    c.bench_function("NameSegment::<BigNumRug>::new(rng)", |b| {
+        b.iter(|| NameSegment::<BigNumRug>::new(&mut thread_rng()));
     });
 }
 
 fn name_accumulator_add(c: &mut Criterion) {
-    let setup = &AccumulatorSetup::from_rsa_2048(&mut thread_rng());
-    c.bench_function("NameAccumulator::add", |b| {
+    let setup = &AccumulatorSetup::<BigNumDig>::from_rsa_2048(&mut thread_rng());
+    c.bench_function("NameAccumulator::<BigNumDig>::add", |b| {
+        b.iter_batched(
+            || NameSegment::<BigNumDig>::new(&mut thread_rng()),
+            |segment| NameAccumulator::<BigNumDig>::empty(setup).add(Some(&segment), setup),
+            BatchSize::SmallInput,
+        )
+    });
+    let setup = &AccumulatorSetup::<BigNumRug>::from_rsa_2048(&mut thread_rng());
+    c.bench_function("NameAccumulator::<BigNumRug>::add", |b| {
         b.iter_batched(
-            || NameSegment::new(&mut thread_rng()),
-            |segment| NameAccumulator::empty(setup).add(Some(&segment), setup),
+            || NameSegment::<BigNumRug>::new(&mut thread_rng()),
+            |segment| NameAccumulator::<BigNumRug>::empty(setup).add(Some(&segment), setup),
             BatchSize::SmallInput,
         )
     });
 }
 
 fn name_accumulator_serialize(c: &mut Criterion) {
-    let setup = &AccumulatorSetup::from_rsa_2048(&mut thread_rng());
-    c.bench_function("NameAccumulator serialization", |b| {
+    let setup = &AccumulatorSetup::<BigNumDig>::from_rsa_2048(&mut thread_rng());
+    c.bench_function("NameAccumulator::<BigNumDig> serialization", |b| {
+        b.iter_batched(
+            || {
+                let segment = NameSegment::<BigNumDig>::new(&mut thread_rng());
+                let mut name = NameAccumulator::<BigNumDig>::empty(setup);
+                name.add(Some(&segment), setup);
+                name
+            },
+            |name| name.into_bytes(),
+            BatchSize::SmallInput,
+        )
+    });
+    let setup = &AccumulatorSetup::<BigNumRug>::from_rsa_2048(&mut thread_rng());
+    c.bench_function("NameAccumulator::<BigNumRug> serialization", |b| {
         b.iter_batched(
             || {
-                let segment = NameSegment::new(&mut thread_rng());
-                let mut name = NameAccumulator::empty(setup);
+                let segment = NameSegment::<BigNumRug>::new(&mut thread_rng());
+                let mut name = NameAccumulator::<BigNumRug>::empty(setup);
                 name.add(Some(&segment), setup);
                 name
             },

wnfs-common/src/utils/test.rs

@@ -7,8 +7,8 @@ use bytes::Bytes;
 use libipld::{
     cbor::DagCborCodec,
     json::DagJsonCodec,
-    prelude::{Decode, Encode},
-    Cid, Ipld,
+    prelude::{Decode, Encode, References},
+    Cid, Ipld, IpldCodec,
 };
 use parking_lot::Mutex;
 use proptest::{
@@ -18,7 +18,7 @@ use proptest::{
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::{
-    collections::{BTreeMap, HashMap},
+    collections::{HashMap, HashSet, VecDeque},
     io::Cursor,
 };
 
@@ -42,6 +42,7 @@ base64_serde_type!(Base64Standard, base64::engine::general_purpose::STANDARD);
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct BlockSnapshot {
+    pub cid: String,
     pub value: Value,
     #[serde(with = "Base64Standard")]
     pub bytes: Bytes,
@@ -59,10 +60,10 @@ pub trait Sampleable {
 impl SnapshotBlockStore {
     pub async fn get_block_snapshot(&self, cid: &Cid) -> Result<BlockSnapshot> {
         let bytes = self.get_block(cid).await?;
-        self.handle_block(cid, &bytes).map(|(_, snapshot)| snapshot)
+        self.handle_block(cid, &bytes)
     }
 
-    pub fn handle_block(&self, cid: &Cid, bytes: &Bytes) -> Result<(String, BlockSnapshot)> {
+    pub fn handle_block(&self, cid: &Cid, bytes: &Bytes) -> Result<BlockSnapshot> {
         let ipld = match cid.codec() {
             CODEC_DAG_CBOR => Ipld::decode(DagCborCodec, &mut Cursor::new(bytes))?,
             CODEC_RAW => match self.block_handlers.lock().get(cid) {
@@ -76,22 +77,34 @@ impl SnapshotBlockStore {
         ipld.encode(DagJsonCodec, &mut json_bytes)?;
 
         let value = serde_json::from_slice(&json_bytes)?;
-        Ok((
-            cid.to_string(),
-            BlockSnapshot {
-                value,
-                bytes: bytes.clone(),
-            },
-        ))
+        Ok(BlockSnapshot {
+            cid: cid.to_string(),
+            value,
+            bytes: bytes.clone(),
+        })
     }
 
-    pub fn get_all_block_snapshots(&self) -> Result<BTreeMap<String, BlockSnapshot>> {
-        self.inner
-            .0
-            .lock()
-            .iter()
-            .map(|(cid, bytes)| self.handle_block(cid, bytes))
-            .collect()
+    pub async fn get_dag_snapshot(&self, root_cid: Cid) -> Result<Vec<BlockSnapshot>> {
+        let mut frontier = VecDeque::from([root_cid]);
+        let mut visited = HashSet::new();
+        let mut snapshots = Vec::new();
+
+        while let Some(cid) = frontier.pop_front() {
+            if !visited.insert(cid) {
+                continue;
+            }
+
+            let snapshot = self.get_block_snapshot(&cid).await?;
+            let codec: IpldCodec = cid.codec().try_into()?;
+            <Ipld as References<IpldCodec>>::references(
+                codec,
+                &mut Cursor::new(&snapshot.bytes),
+                &mut frontier,
+            )?;
+            snapshots.push(snapshot);
+        }
+
+        Ok(snapshots)
     }
 
     pub fn add_block_handler(&mut self, cid: Cid, f: BlockHandler) {

wnfs-hamt/src/snapshots/wnfs_hamt__hamt__snapshot_tests__hamt.snap

@@ -3,6 +3,7 @@ source: wnfs-hamt/src/hamt.rs
 expression: hamt
 ---
 {
+  "cid": "bafyr4iaveogpzpomn7qakjyrq6nynt6b2pemjiejr7unlxhalfdciw74qa",
   "value": {
     "root": [
       {

wnfs-hamt/src/snapshots/wnfs_hamt__node__snapshot_tests__node.snap

@@ -3,6 +3,7 @@ source: wnfs-hamt/src/node.rs
 expression: node
 ---
 {
+  "cid": "bafyr4igc6oq6uq55xmd4v6dheqdydlyzexhqs5zfewtudpwsinanqmylyq",
   "value": [
     {
       "/": {

wnfs-hamt/src/snapshots/wnfs_hamt__pointer__snapshot_tests__pointer.snap

@@ -3,6 +3,7 @@ source: wnfs-hamt/src/pointer.rs
 expression: ptr
 ---
 {
+  "cid": "bafyr4icjrohmzg3axrk7guxd42zmfwhdx7ptio3n4lsg23hywjrmjgpmpa",
   "value": [
     [
       "James",

wnfs-nameaccumulator/Cargo.toml

@@ -21,11 +21,12 @@ anyhow = "1.0"
 async-trait = "0.1"
 blake3 = { version = "1.4", features = ["traits-preview"] }
 libipld = { version = "0.16", features = ["dag-cbor", "derive", "serde-codec"] }
-num-bigint-dig = { version = "0.8.2", features = ["prime", "zeroize"] }
+num-bigint-dig = { version = "0.8.2", features = ["prime", "zeroize"], optional = true }
 num-integer = "0.1.45"
 num-traits = "0.2.15"
 once_cell = "1.0"
 rand_core = "0.6"
+rug = { version = "1.22", optional = true, default-features = false, features = ["rand", "integer", "num-traits"] }
 serde = { version = "1.0", features = ["rc"] }
 serde_bytes = "0.11.9"
 thiserror = "1.0"
@@ -42,3 +43,8 @@ rand_chacha = "0.3"
 serde_json = "1.0.103"
 test-strategy = "0.3"
 wnfs-common = { path = "../wnfs-common", features = ["test_utils"] }
+
+[features]
+default = ["num-bigint-dig"]
+rug = ["dep:rug"]
+num-bigint-dig = ["dep:num-bigint-dig"]

wnfs-nameaccumulator/README.md

@@ -82,3 +82,22 @@ verification.add(&name_base, &accum_note, &proof_note.part)?;
 verification.add(&name_base, &accum_image, &proof_image.part)?;
 verification.verify(&batched_proof)?;
 ```
+
+## The `rug` feature
+
+This enables a different backend for big unsigned integer arithmetic, based on the [rug crate] (which is based on the [GNU multiprecision library], also abbreviated GMP).
+
+It is roughly 2x faster than the `num-bigint-dig` implementation when building for release, but as a bonus is also fast in debug builds (e.g. during tests) due to rug containing a statically linked release build of GMP.
+
+However, it doesn't work in Wasm and it should be noted that GMP is licensed as [LGPLv3].
+
+If you depend on the `wnfs` crate, but want to use the `rug` backend for your application, then simply add a `wnfs-nameaccumulator` as a dependency and enable its `rug` feature. This will make `wnfs` use a version of `wnfs-nameaccumulator` with `rug` enabled:
+
+```toml
+wnfs-nameaccumulator = { version = "*", default-features = false, features = ["rug"] }
+```
+
+
+[rug crate]: https://crates.io/crates/rug
+[GNU multiprecision library]: https://gmplib.org/
+[LGPLv3]: https://www.gnu.org/licenses/lgpl-3.0.en.html

wnfs-nameaccumulator/proptest-regressions/name.txt

@@ -5,3 +5,4 @@
 # It is recommended to check this file in to source control so that
 # everyone who runs the test benefits from these saved cases.
 cc 981ff5c92ddd0a53eb8569b94c7aa0184603de547961d5562c40e1c8643a7275 # shrinks to input = _PaddedBiguintEncodingRoundtripsArgs { num: 1 }
+cc 96d340fe66ed56e3479dd02bbccc0a500237d7720fb1488557806d6a61972205 # shrinks to input = _BatchProofsArgs { do_batch_step: [false, false, true, true], do_verify_step: [false, true, true, false], seed: 0 }

wnfs-nameaccumulator/src/error.rs

@@ -16,4 +16,7 @@ pub enum VerificationError {
 
     #[error("NameAccumulator batched proof validation failed")]
     ValidationFailed,
+
+    #[error("Couldn't invert base accumulator state")]
+    NoInverse,
 }