From deba62721ca33579d0642098521b3b5555a1db7e Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Wed, 27 Dec 2023 22:23:29 -0300 Subject: [PATCH 01/12] WIP raw pointers and store --- src/lem/mod.rs | 2 + src/lem/raw_pointers.rs | 127 ++++ src/lem/raw_store.rs | 1228 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 1357 insertions(+) create mode 100644 src/lem/raw_pointers.rs create mode 100644 src/lem/raw_store.rs diff --git a/src/lem/mod.rs b/src/lem/mod.rs index 2634353ff5..836ef4c4ba 100644 --- a/src/lem/mod.rs +++ b/src/lem/mod.rs @@ -66,6 +66,8 @@ mod macros; pub mod multiframe; mod path; pub mod pointers; +pub mod raw_pointers; +pub mod raw_store; mod slot; pub mod store; mod var_map; diff --git a/src/lem/raw_pointers.rs b/src/lem/raw_pointers.rs new file mode 100644 index 0000000000..ab3c440f62 --- /dev/null +++ b/src/lem/raw_pointers.rs @@ -0,0 +1,127 @@ +use super::Tag; +use crate::tag::ExprTag::{Cons, Fun, Nil, Num, Str, Sym}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub enum RawPtr { + Atom(usize), + Hash3(usize), + Hash4(usize), + Hash6(usize), + Hash8(usize), +} + +impl RawPtr { + #[inline] + pub fn is_hash(&self) -> bool { + matches!( + self, + RawPtr::Hash3(..) | RawPtr::Hash4(..) | RawPtr::Hash6(..) | RawPtr::Hash8(..) + ) + } + + #[inline] + pub fn get_atom(&self) -> Option { + match self { + RawPtr::Atom(x) => Some(*x), + _ => None, + } + } + + #[inline] + pub fn get_hash3(&self) -> Option { + match self { + RawPtr::Hash3(x) => Some(*x), + _ => None, + } + } + + #[inline] + pub fn get_hash4(&self) -> Option { + match self { + RawPtr::Hash4(x) => Some(*x), + _ => None, + } + } + + #[inline] + pub fn get_hash6(&self) -> Option { + match self { + RawPtr::Hash6(x) => Some(*x), + _ => None, + } + } + + #[inline] + pub fn get_hash8(&self) -> Option { + match self { + RawPtr::Hash8(x) => Some(*x), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub struct Ptr { + tag: Tag, + pay: RawPtr, +} + +impl Ptr { + pub fn new(tag: Tag, pay: RawPtr) -> Self { + Ptr { tag, pay } + } + + pub fn tag(&self) -> &Tag { + &self.tag + } + + pub fn pay(&self) -> &RawPtr { + &self.pay + } + + #[inline] + pub fn has_tag(&self, tag: &Tag) -> bool { + self.tag() == tag + } + + #[inline] + pub fn has_tag_in(&self, tags: &[Tag]) -> bool { + tags.contains(self.tag()) + } + + #[inline] + pub fn is_sym(&self) -> bool { + self.has_tag(&Tag::Expr(Sym)) + } + + #[inline] + pub fn is_num(&self) -> bool { + self.has_tag(&Tag::Expr(Num)) + } + + #[inline] + pub fn is_str(&self) -> bool { + self.has_tag(&Tag::Expr(Str)) + } + + #[inline] + pub fn is_fun(&self) -> bool { + self.has_tag(&Tag::Expr(Fun)) + } + + #[inline] + pub fn is_nil(&self) -> bool { + self.has_tag(&Tag::Expr(Nil)) + } + + #[inline] + pub fn is_list(&self) -> bool { + self.has_tag_in(&[Tag::Expr(Cons), Tag::Expr(Nil)]) + } + + #[inline] + pub fn cast(self, tag: Tag) -> Self { + Ptr { tag, pay: self.pay } + } +} diff --git a/src/lem/raw_store.rs b/src/lem/raw_store.rs new file mode 100644 index 0000000000..834cfcf840 --- /dev/null +++ b/src/lem/raw_store.rs @@ -0,0 +1,1228 @@ +use anyhow::{bail, Context, Result}; +use arc_swap::ArcSwap; +use bellpepper::util_cs::witness_cs::SizedWitness; +use elsa::{ + sync::index_set::FrozenIndexSet, + sync::{FrozenMap, FrozenVec}, +}; +use indexmap::IndexSet; +use neptune::Poseidon; +use nom::{sequence::preceded, Parser}; +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use std::{cell::RefCell, rc::Rc, sync::Arc}; + +use crate::{ + field::{FWrap, LurkField}, + hash::{InversePoseidonCache, PoseidonCache}, + lem::Tag, + parser::{syntax, Error, Span}, + state::{lurk_sym, user_sym, State}, + symbol::Symbol, + syntax::Syntax, + tag::ContTag::{ + self, Binop, Binop2, Call, Call0, Call2, Dummy, Emit, If, Let, LetRec, Lookup, Outermost, + Tail, Terminal, Unop, + }, + tag::ExprTag::{Char, Comm, Cons, Cproc, Fun, Key, Nil, Num, Str, Sym, Thunk, U64}, + tag::Tag as TagTrait, +}; + +use super::raw_pointers::{Ptr, RawPtr}; + +#[derive(Debug)] +pub struct Store { + f_elts: FrozenIndexSet>>, + hash3: FrozenIndexSet>, + hash4: FrozenIndexSet>, + hash6: FrozenIndexSet>, + hash8: FrozenIndexSet>, + + string_ptr_cache: FrozenMap>, + symbol_ptr_cache: FrozenMap>, + + ptr_string_cache: FrozenMap, + ptr_symbol_cache: FrozenMap>, + + comms: FrozenMap, Box<(F, Ptr)>>, // hash -> (secret, src) + + pub poseidon_cache: PoseidonCache, + pub inverse_poseidon_cache: InversePoseidonCache, + + dehydrated: ArcSwap>>, + z_cache: FrozenMap>>, + inverse_z_cache: FrozenMap, Box>, + + // cached indices for the hashes of 3, 4, 6 and 8 padded zeros + pub hash3zeros_idx: usize, + pub hash4zeros_idx: usize, + pub hash6zeros_idx: usize, + pub hash8zeros_idx: usize, +} + +impl Default for Store { + fn default() -> Self { + let poseidon_cache = PoseidonCache::default(); + let hash3zeros = poseidon_cache.hash3(&[F::ZERO; 3]); + let hash4zeros = poseidon_cache.hash4(&[F::ZERO; 4]); + let hash6zeros = poseidon_cache.hash6(&[F::ZERO; 6]); + let hash8zeros = poseidon_cache.hash8(&[F::ZERO; 8]); + + let f_elts = FrozenIndexSet::default(); + let (hash3zeros_idx, _) = f_elts.insert_probe(FWrap(hash3zeros).into()); + let (hash4zeros_idx, _) = f_elts.insert_probe(FWrap(hash4zeros).into()); + let (hash6zeros_idx, _) = f_elts.insert_probe(FWrap(hash6zeros).into()); + let (hash8zeros_idx, _) = f_elts.insert_probe(FWrap(hash8zeros).into()); + + Self { + f_elts, + hash3: Default::default(), + hash4: Default::default(), + hash6: Default::default(), + hash8: Default::default(), + string_ptr_cache: Default::default(), + symbol_ptr_cache: Default::default(), + ptr_string_cache: Default::default(), + ptr_symbol_cache: Default::default(), + comms: Default::default(), + poseidon_cache, + inverse_poseidon_cache: Default::default(), + dehydrated: Default::default(), + z_cache: Default::default(), + inverse_z_cache: Default::default(), + hash3zeros_idx, + hash4zeros_idx, + hash6zeros_idx, + hash8zeros_idx, + } + } +} + +impl Store { + /// Cost of poseidon hash with arity 3, including the input + #[inline] + pub fn hash3_cost(&self) -> usize { + Poseidon::new(self.poseidon_cache.constants.c3()).num_aux() + 1 + } + + /// Cost of poseidon hash with arity 4, including the input + #[inline] + pub fn hash4_cost(&self) -> usize { + Poseidon::new(self.poseidon_cache.constants.c4()).num_aux() + 1 + } + + /// Cost of poseidon hash with arity 6, including the input + #[inline] + pub fn hash6_cost(&self) -> usize { + Poseidon::new(self.poseidon_cache.constants.c6()).num_aux() + 1 + } + + /// Cost of poseidon hash with arity 8, including the input + #[inline] + pub fn hash8_cost(&self) -> usize { + Poseidon::new(self.poseidon_cache.constants.c8()).num_aux() + 1 + } + + /// Retrieves the hash of 3 padded zeros + #[inline] + pub fn hash3zeros(&self) -> &F { + self.expect_f(self.hash3zeros_idx) + } + + /// Retrieves the hash of 4 padded zeros + #[inline] + pub fn hash4zeros(&self) -> &F { + self.expect_f(self.hash4zeros_idx) + } + + /// Retrieves the hash of 6 padded zeros + #[inline] + pub fn hash6zeros(&self) -> &F { + self.expect_f(self.hash6zeros_idx) + } + + /// Retrieves the hash of 8 padded zeros + #[inline] + pub fn hash8zeros(&self) -> &F { + self.expect_f(self.hash8zeros_idx) + } + + #[inline] + pub fn intern_f(&self, f: F) -> (usize, bool) { + self.f_elts.insert_probe(Box::new(FWrap(f))) + } + + /// Creates an atom `Ptr` which points to a cached element of the finite + /// field `F` + pub fn intern_atom(&self, f: F) -> RawPtr { + let (idx, _) = self.intern_f(f); + RawPtr::Atom(idx) + } + + /// Creates a `Ptr` that's a parent of 3 children + pub fn intern_3_ptrs(&self, ptrs: [RawPtr; 3]) -> RawPtr { + let (idx, inserted) = self.hash3.insert_probe(Box::new(ptrs)); + let ptr = RawPtr::Hash3(idx); + if inserted { + // this is for `hydrate_z_cache` + self.dehydrated.load().push(Box::new(ptr)); + } + ptr + } + + /// Similar to `intern_3_ptrs` but doesn't add the resulting pointer to + /// `dehydrated`. This function is used when converting a `ZStore` to a + /// `Store`. + pub fn intern_3_ptrs_hydrated(&self, ptrs: [RawPtr; 3], z: FWrap) -> RawPtr { + let (idx, _) = self.hash3.insert_probe(Box::new(ptrs)); + let ptr = RawPtr::Hash3(idx); + self.z_cache.insert(ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(ptr)); + ptr + } + + /// Creates a `Ptr` that's a parent of 4 children + pub fn intern_4_ptrs(&self, ptrs: [RawPtr; 4]) -> RawPtr { + let (idx, inserted) = self.hash4.insert_probe(Box::new(ptrs)); + let ptr = RawPtr::Hash4(idx); + if inserted { + // this is for `hydrate_z_cache` + self.dehydrated.load().push(Box::new(ptr)); + } + ptr + } + + /// Similar to `intern_4_ptrs` but doesn't add the resulting pointer to + /// `dehydrated`. This function is used when converting a `ZStore` to a + /// `Store`. + pub fn intern_4_ptrs_hydrated(&self, ptrs: [RawPtr; 4], z: FWrap) -> RawPtr { + let (idx, _) = self.hash4.insert_probe(Box::new(ptrs)); + let ptr = RawPtr::Hash4(idx); + self.z_cache.insert(ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(ptr)); + ptr + } + + /// Creates a `Ptr` that's a parent of 6 children + pub fn intern_6_ptrs(&self, ptrs: [RawPtr; 6]) -> RawPtr { + let (idx, inserted) = self.hash6.insert_probe(Box::new(ptrs)); + let ptr = RawPtr::Hash6(idx); + if inserted { + // this is for `hydrate_z_cache` + self.dehydrated.load().push(Box::new(ptr)); + } + ptr + } + + /// Similar to `intern_6_ptrs` but doesn't add the resulting pointer to + /// `dehydrated`. This function is used when converting a `ZStore` to a + /// `Store`. + pub fn intern_6_ptrs_hydrated(&self, ptrs: [RawPtr; 6], z: FWrap) -> RawPtr { + let (idx, _) = self.hash6.insert_probe(Box::new(ptrs)); + let ptr = RawPtr::Hash6(idx); + self.z_cache.insert(ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(ptr)); + ptr + } + + /// Creates a `Ptr` that's a parent of 8 children + pub fn intern_8_ptrs(&self, ptrs: [RawPtr; 8]) -> RawPtr { + let (idx, inserted) = self.hash8.insert_probe(Box::new(ptrs)); + let ptr = RawPtr::Hash8(idx); + if inserted { + // this is for `hydrate_z_cache` + self.dehydrated.load().push(Box::new(ptr)); + } + ptr + } + + /// Similar to `intern_8_ptrs` but doesn't add the resulting pointer to + /// `dehydrated`. This function is used when converting a `ZStore` to a + /// `Store`. + pub fn intern_8_ptrs_hydrated(&self, ptrs: [RawPtr; 8], z: FWrap) -> RawPtr { + let (idx, _) = self.hash8.insert_probe(Box::new(ptrs)); + let ptr = RawPtr::Hash8(idx); + self.z_cache.insert(ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(ptr)); + ptr + } + + #[inline] + pub fn fetch_f(&self, idx: usize) -> Option<&F> { + self.f_elts.get_index(idx).map(|fw| &fw.0) + } + + #[inline] + pub fn fetch_3_ptrs(&self, idx: usize) -> Option<&[RawPtr; 3]> { + self.hash3.get_index(idx) + } + + #[inline] + pub fn fetch_4_ptrs(&self, idx: usize) -> Option<&[RawPtr; 4]> { + self.hash4.get_index(idx) + } + + #[inline] + pub fn fetch_6_ptrs(&self, idx: usize) -> Option<&[RawPtr; 6]> { + self.hash6.get_index(idx) + } + + #[inline] + pub fn fetch_8_ptrs(&self, idx: usize) -> Option<&[RawPtr; 8]> { + self.hash8.get_index(idx) + } + + #[inline] + pub fn expect_f(&self, idx: usize) -> &F { + self.fetch_f(idx).expect("Index missing from f_elts") + } + + #[inline] + pub fn expect_3_ptrs(&self, idx: usize) -> &[RawPtr; 3] { + self.fetch_3_ptrs(idx).expect("Index missing from store") + } + + #[inline] + pub fn expect_4_ptrs(&self, idx: usize) -> &[RawPtr; 4] { + self.fetch_4_ptrs(idx).expect("Index missing from store") + } + + #[inline] + pub fn expect_6_ptrs(&self, idx: usize) -> &[RawPtr; 6] { + self.fetch_6_ptrs(idx).expect("Index missing from store") + } + + #[inline] + pub fn expect_8_ptrs(&self, idx: usize) -> &[RawPtr; 8] { + self.fetch_8_ptrs(idx).expect("Index missing from store") + } + + #[inline] + pub fn tag(&self, tag: Tag) -> RawPtr { + self.intern_atom(tag.to_field()) + } + + #[inline] + pub fn fetch_tag(&self, ptr: &RawPtr) -> Option { + let idx = ptr.get_atom()?; + let f = self.fetch_f(idx)?; + TagTrait::from_field(f) + } + + pub fn raw_to_ptr(&self, tag: &RawPtr, pay: &RawPtr) -> Option { + let tag = self.fetch_tag(tag)?; + Some(Ptr::new(tag, *pay)) + } + + #[inline] + pub fn num(&self, f: F) -> Ptr { + Ptr::new(Tag::Expr(Num), self.intern_atom(f)) + } + + #[inline] + pub fn num_u64(&self, u: u64) -> Ptr { + Ptr::new(Tag::Expr(Num), self.intern_atom(F::from_u64(u))) + } + + #[inline] + pub fn u64(&self, u: u64) -> Ptr { + Ptr::new(Tag::Expr(U64), self.intern_atom(F::from_u64(u))) + } + + #[inline] + pub fn char(&self, c: char) -> Ptr { + Ptr::new(Tag::Expr(Char), self.intern_atom(F::from_char(c))) + } + + #[inline] + pub fn comm(&self, hash: F) -> Ptr { + Ptr::new(Tag::Expr(Comm), self.intern_atom(hash)) + } + + #[inline] + pub fn zero(&self) -> RawPtr { + self.intern_atom(F::ZERO) + } + + pub fn is_zero(&self, ptr: &RawPtr) -> bool { + match ptr { + RawPtr::Atom(idx) => self.fetch_f(*idx) == Some(&F::ZERO), + _ => false, + } + } + + #[inline] + pub fn dummy(&self) -> RawPtr { + self.zero() + } + + /// Creates an atom pointer from a `ZPtr`, with its hash. Hashing + /// such pointer will result on the same original `ZPtr` + #[inline] + pub fn opaque(&self, z: FWrap) -> RawPtr { + self.intern_atom(z.0) + } + + pub fn intern_string(&self, s: &str) -> Ptr { + if let Some(ptr) = self.string_ptr_cache.get(s) { + *ptr + } else { + let pay = s.chars().rev().fold(self.zero(), |acc, c| { + let ptrs = [ + self.tag(Tag::Expr(Char)), + *self.char(c).pay(), + self.tag(Tag::Expr(Str)), + acc, + ]; + self.intern_4_ptrs(ptrs) + }); + let ptr = Ptr::new(Tag::Expr(Str), pay); + self.string_ptr_cache.insert(s.to_string(), Box::new(ptr)); + self.ptr_string_cache.insert(ptr, s.to_string()); + ptr + } + } + + pub fn fetch_string(&self, ptr: &Ptr) -> Option { + if let Some(str) = self.ptr_string_cache.get(ptr) { + Some(str.to_string()) + } else { + let mut string = String::new(); + let mut ptr = *ptr; + if *ptr.tag() != Tag::Expr(Str) { + return None; + } + loop { + match *ptr.pay() { + RawPtr::Atom(idx) => { + if self.fetch_f(idx)? == &F::ZERO { + self.ptr_string_cache.insert(ptr, string.clone()); + return Some(string); + } else { + return None; + } + } + RawPtr::Hash4(idx) => { + let [car_tag, car, cdr_tag, cdr] = self.fetch_4_ptrs(idx)?; + assert_eq!(*car_tag, self.tag(Tag::Expr(Char))); + assert_eq!(*cdr_tag, self.tag(Tag::Expr(Str))); + match car { + RawPtr::Atom(idx) => { + let f = self.fetch_f(*idx)?; + string.push(f.to_char().expect("malformed char pointer")); + ptr = Ptr::new(Tag::Expr(Str), *cdr) + } + _ => return None, + } + } + _ => return None, + } + } + } + } + + pub fn intern_symbol_path(&self, path: &[String]) -> RawPtr { + path.iter().fold(self.zero(), |acc, s| { + let ptrs = [ + self.tag(Tag::Expr(Str)), + *self.intern_string(s).pay(), + self.tag(Tag::Expr(Sym)), + acc, + ]; + self.intern_4_ptrs(ptrs) + }) + } + + pub fn intern_symbol(&self, sym: &Symbol) -> Ptr { + if let Some(ptr) = self.symbol_ptr_cache.get(sym) { + *ptr + } else { + let path_ptr = self.intern_symbol_path(sym.path()); + let sym_ptr = if sym == &lurk_sym("nil") { + Ptr::new(Tag::Expr(Nil), path_ptr) + } else if sym.is_keyword() { + Ptr::new(Tag::Expr(Key), path_ptr) + } else { + Ptr::new(Tag::Expr(Sym), path_ptr) + }; + self.symbol_ptr_cache.insert(sym.clone(), Box::new(sym_ptr)); + self.ptr_symbol_cache.insert(sym_ptr, Box::new(sym.clone())); + sym_ptr + } + } + + /// Fetches a symbol path whose interning returned the provided `idx` + fn fetch_symbol_path(&self, mut idx: usize) -> Option> { + let mut path = vec![]; + loop { + let [car_tag, car, cdr_tag, cdr] = self.fetch_4_ptrs(idx)?; + assert_eq!(*car_tag, self.tag(Tag::Expr(Str))); + assert_eq!(*cdr_tag, self.tag(Tag::Expr(Sym))); + let string = self.fetch_string(&Ptr::new(Tag::Expr(Str), *car))?; + path.push(string); + match cdr { + RawPtr::Atom(idx) => { + if self.fetch_f(*idx)? == &F::ZERO { + path.reverse(); + return Some(path); + } else { + return None; + } + } + RawPtr::Hash4(idx_cdr) => idx = *idx_cdr, + _ => return None, + } + } + } + + pub fn fetch_symbol(&self, ptr: &Ptr) -> Option { + if let Some(sym) = self.ptr_symbol_cache.get(ptr) { + Some(sym.clone()) + } else { + match (ptr.tag(), ptr.pay()) { + (Tag::Expr(Sym), RawPtr::Atom(idx)) => { + if self.fetch_f(*idx)? == &F::ZERO { + let sym = Symbol::root_sym(); + self.ptr_symbol_cache.insert(*ptr, Box::new(sym.clone())); + Some(sym) + } else { + None + } + } + (Tag::Expr(Key), RawPtr::Atom(idx)) => { + if self.fetch_f(*idx)? == &F::ZERO { + let key = Symbol::root_key(); + self.ptr_symbol_cache.insert(*ptr, Box::new(key.clone())); + Some(key) + } else { + None + } + } + (Tag::Expr(Sym | Nil), RawPtr::Hash4(idx)) => { + let path = self.fetch_symbol_path(*idx)?; + let sym = Symbol::sym_from_vec(path); + self.ptr_symbol_cache.insert(*ptr, Box::new(sym.clone())); + Some(sym) + } + (Tag::Expr(Key), RawPtr::Hash4(idx)) => { + let path = self.fetch_symbol_path(*idx)?; + let key = Symbol::key_from_vec(path); + self.ptr_symbol_cache.insert(*ptr, Box::new(key.clone())); + Some(key) + } + _ => None, + } + } + } + + pub fn fetch_sym(&self, ptr: &Ptr) -> Option { + if ptr.tag() == &Tag::Expr(Sym) { + self.fetch_symbol(ptr) + } else { + None + } + } + + pub fn fetch_key(&self, ptr: &Ptr) -> Option { + if ptr.tag() == &Tag::Expr(Key) { + self.fetch_symbol(ptr) + } else { + None + } + } + + #[inline] + pub fn intern_lurk_symbol(&self, name: &str) -> Ptr { + self.intern_symbol(&lurk_sym(name)) + } + + #[inline] + pub fn intern_nil(&self) -> Ptr { + self.intern_lurk_symbol("nil") + } + + #[inline] + pub fn intern_user_symbol(&self, name: &str) -> Ptr { + self.intern_symbol(&user_sym(name)) + } + + #[inline] + pub fn key(&self, name: &str) -> Ptr { + self.intern_symbol(&Symbol::key(&[name.to_string()])) + } + + #[inline] + pub fn add_comm(&self, hash: F, secret: F, payload: Ptr) { + self.comms.insert(FWrap(hash), Box::new((secret, payload))); + } + + #[inline] + pub fn hide(&self, secret: F, payload: Ptr) -> Ptr { + self.comm(self.hide_ptr(secret, payload)) + } + + #[inline] + pub fn commit(&self, payload: Ptr) -> Ptr { + self.hide(F::NON_HIDING_COMMITMENT_SECRET, payload) + } + + #[inline] + pub fn open(&self, hash: F) -> Option<&(F, Ptr)> { + self.comms.get(&FWrap(hash)) + } + + pub fn hide_ptr(&self, secret: F, payload: Ptr) -> F { + let hash = self.poseidon_cache.hash3(&[ + secret, + payload.tag().to_field(), + self.hash_ptr(payload.pay()).0, + ]); + self.add_comm(hash, secret, payload); + hash + } + + #[inline] + pub fn cons(&self, car: Ptr, cdr: Ptr) -> Ptr { + let ptrs = [ + self.tag(*car.tag()), + *car.pay(), + self.tag(*cdr.tag()), + *cdr.pay(), + ]; + Ptr::new(Tag::Expr(Cons), self.intern_4_ptrs(ptrs)) + } + + #[inline] + pub fn intern_fun(&self, arg: Ptr, body: Ptr, env: Ptr) -> Ptr { + let ptrs = [ + self.tag(*arg.tag()), + *arg.pay(), + self.tag(*body.tag()), + *body.pay(), + self.tag(*env.tag()), + *env.pay(), + self.tag(Tag::Expr(Nil)), + self.dummy(), + ]; + Ptr::new(Tag::Expr(Fun), self.intern_8_ptrs(ptrs)) + } + + #[inline] + pub fn cont_outermost(&self) -> Ptr { + Ptr::new(Tag::Cont(Outermost), RawPtr::Atom(self.hash8zeros_idx)) + } + + #[inline] + pub fn cont_error(&self) -> Ptr { + Ptr::new(Tag::Cont(ContTag::Error), RawPtr::Atom(self.hash8zeros_idx)) + } + + #[inline] + pub fn cont_terminal(&self) -> Ptr { + Ptr::new(Tag::Cont(Terminal), RawPtr::Atom(self.hash8zeros_idx)) + } + + pub fn car_cdr(&self, ptr: &Ptr) -> Result<(Ptr, Ptr)> { + match ptr.tag() { + Tag::Expr(Nil) => { + let nil = self.intern_nil(); + Ok((nil, nil)) + } + Tag::Expr(Cons) => { + let Some(idx) = ptr.pay().get_hash4() else { + bail!("malformed cons pointer") + }; + match self.fetch_4_ptrs(idx) { + Some([car_tag, car, cdr_tag, cdr]) => { + let car_ptr = self.raw_to_ptr(car_tag, car).context("Not a pointer")?; + let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr).context("Not a pointer")?; + Ok((car_ptr, cdr_ptr)) + } + None => bail!("car/cdr not found"), + } + } + Tag::Expr(Str) => { + if self.is_zero(ptr.pay()) { + let nil_str = Ptr::new(Tag::Expr(Str), self.zero()); + Ok((self.intern_nil(), nil_str)) + } else { + let Some(idx) = ptr.pay().get_hash4() else { + bail!("malformed str pointer") + }; + match self.fetch_4_ptrs(idx) { + Some([car_tag, car, cdr_tag, cdr]) => { + let car_ptr = self.raw_to_ptr(car_tag, car).context("Not a pointer")?; + let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr).context("Not a pointer")?; + Ok((car_ptr, cdr_ptr)) + } + None => bail!("car/cdr not found"), + } + } + } + _ => bail!("invalid pointer to extract car/cdr from"), + } + } + + /// Interns a sequence of pointers as a cons-list. The terminating element + /// defaults to `nil` if `last` is `None` + fn intern_list(&self, elts: Vec, last: Option) -> Ptr { + elts.into_iter() + .rev() + .fold(last.unwrap_or_else(|| self.intern_nil()), |acc, elt| { + self.cons(acc, elt) + }) + } + + /// Interns a sequence of pointers as a proper (`nil`-terminated) cons-list + #[inline] + pub fn list(&self, elts: Vec) -> Ptr { + self.intern_list(elts, None) + } + + /// Interns a sequence of pointers as an improper cons-list whose last + /// element is `last` + #[inline] + pub fn improper_list(&self, elts: Vec, last: Ptr) -> Ptr { + self.intern_list(elts, Some(last)) + } + + /// Fetches a cons list that was interned. If the list is improper, the second + /// element of the returned pair will carry the improper terminating value + pub fn fetch_list(&self, ptr: &Ptr) -> Option<(Vec, Option)> { + if *ptr == self.intern_nil() { + return Some((vec![], None)); + } + match (ptr.tag(), ptr.pay()) { + (Tag::Expr(Nil), _) => panic!("Malformed nil expression"), + (Tag::Expr(Cons), RawPtr::Hash4(mut idx)) => { + let mut list = vec![]; + let mut last = None; + while let Some([car_tag, car, cdr_tag, cdr]) = self.fetch_4_ptrs(idx) { + let car_ptr = self.raw_to_ptr(car_tag, car)?; + let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr)?; + list.push(car_ptr); + match cdr_ptr.tag() { + Tag::Expr(Nil) => break, + Tag::Expr(Cons) => { + idx = cdr.get_hash4()?; + } + _ => { + last = Some(cdr_ptr); + break; + } + } + } + Some((list, last)) + } + _ => None, + } + } + + pub fn intern_syntax(&self, syn: Syntax) -> Ptr { + match syn { + Syntax::Num(_, x) => self.num(x.into_scalar()), + Syntax::UInt(_, x) => self.u64(x.into()), + Syntax::Char(_, x) => self.char(x), + Syntax::Symbol(_, x) => self.intern_symbol(&x), + Syntax::String(_, x) => self.intern_string(&x), + Syntax::Quote(_, x) => self.list(vec![ + self.intern_symbol(&lurk_sym("quote")), + self.intern_syntax(*x), + ]), + Syntax::List(_, xs) => { + self.list(xs.into_iter().map(|x| self.intern_syntax(x)).collect()) + } + Syntax::Improper(_, xs, y) => self.improper_list( + xs.into_iter().map(|x| self.intern_syntax(x)).collect(), + self.intern_syntax(*y), + ), + } + } + + pub fn read(&self, state: Rc>, input: &str) -> Result { + match preceded( + syntax::parse_space, + syntax::parse_syntax(state, false, false), + ) + .parse(Span::new(input)) + { + Ok((_, x)) => Ok(self.intern_syntax(x)), + Err(e) => bail!("{}", e), + } + } + + pub fn read_maybe_meta<'a>( + &self, + state: Rc>, + input: &'a str, + ) -> Result<(usize, Span<'a>, Ptr, bool), Error> { + match preceded(syntax::parse_space, syntax::parse_maybe_meta(state, false)) + .parse(input.into()) + { + Ok((i, Some((is_meta, x)))) => { + let from_offset = x + .get_pos() + .get_from_offset() + .expect("Parsed syntax should have its Pos set"); + Ok((from_offset, i, self.intern_syntax(x), is_meta)) + } + Ok((_, None)) => Err(Error::NoInput), + Err(e) => Err(Error::Syntax(format!("{}", e))), + } + } + + #[inline] + pub fn read_with_default_state(&self, input: &str) -> Result { + self.read(State::init_lurk_state().rccell(), input) + } + + /// Recursively hashes the children of a `Ptr` in order to obtain its + /// corresponding `ZPtr`. While traversing a `Ptr` tree, it consults the + /// cache of `Ptr`s that have already been hydrated and also populates this + /// cache for the new `Ptr`s. + /// + /// Warning: without cache hits, this function might blow up Rust's recursion + /// depth limit. This limitation is circumvented by calling `hydrate_z_cache` + /// beforehand or by using `hash_ptr` instead, which is slightly slower. + fn hash_ptr_unsafe(&self, ptr: &RawPtr) -> FWrap { + match ptr { + RawPtr::Atom(idx) => FWrap(*self.expect_f(*idx)), + RawPtr::Hash3(idx) => { + if let Some(z) = self.z_cache.get(ptr) { + *z + } else { + let children_ptrs = self.expect_3_ptrs(*idx); + let mut children_zs = [F::ZERO; 3]; + for (idx, child_ptr) in children_ptrs.iter().enumerate() { + children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; + } + let z = FWrap(self.poseidon_cache.hash3(&children_zs)); + self.z_cache.insert(*ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(*ptr)); + z + } + } + RawPtr::Hash4(idx) => { + if let Some(z) = self.z_cache.get(ptr) { + *z + } else { + let children_ptrs = self.expect_4_ptrs(*idx); + let mut children_zs = [F::ZERO; 4]; + for (idx, child_ptr) in children_ptrs.iter().enumerate() { + children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; + } + let z = FWrap(self.poseidon_cache.hash4(&children_zs)); + self.z_cache.insert(*ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(*ptr)); + z + } + } + RawPtr::Hash6(idx) => { + if let Some(z) = self.z_cache.get(ptr) { + *z + } else { + let children_ptrs = self.expect_6_ptrs(*idx); + let mut children_zs = [F::ZERO; 6]; + for (idx, child_ptr) in children_ptrs.iter().enumerate() { + children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; + } + let z = FWrap(self.poseidon_cache.hash6(&children_zs)); + self.z_cache.insert(*ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(*ptr)); + z + } + } + RawPtr::Hash8(idx) => { + if let Some(z) = self.z_cache.get(ptr) { + *z + } else { + let children_ptrs = self.expect_8_ptrs(*idx); + let mut children_zs = [F::ZERO; 8]; + for (idx, child_ptr) in children_ptrs.iter().enumerate() { + children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; + } + let z = FWrap(self.poseidon_cache.hash8(&children_zs)); + self.z_cache.insert(*ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(*ptr)); + z + } + } + } + } + + /// Hashes pointers in parallel, consuming chunks of length 256, which is a + /// reasonably safe limit. The danger of longer chunks is that the rightmost + /// pointers are the ones which are more likely to reach the recursion depth + /// limit in `hash_ptr_unsafe`. So we move in smaller chunks from left to + /// right, populating the `z_cache`, which can rescue `hash_ptr_unsafe` from + /// dangerously deep recursions + fn hydrate_z_cache_with_ptrs(&self, ptrs: &[&RawPtr]) { + ptrs.chunks(256).for_each(|chunk| { + chunk.par_iter().for_each(|ptr| { + self.hash_ptr_unsafe(ptr); + }); + }); + } + + /// Hashes enqueued `Ptr` trees from the bottom to the top, avoiding deep + /// recursions in `hash_ptr`. Resets the `dehydrated` queue afterwards. + pub fn hydrate_z_cache(&self) { + self.hydrate_z_cache_with_ptrs(&self.dehydrated.load().iter().collect::>()); + self.dehydrated.swap(Arc::new(FrozenVec::default())); + } + + /// Whether the length of the dehydrated queue is within the safe limit. + /// Note: these values are experimental and may be machine dependant. + #[inline] + fn is_below_safe_threshold(&self) -> bool { + if cfg!(debug_assertions) { + // not release mode + self.dehydrated.load().len() < 443 + } else { + // release mode + self.dehydrated.load().len() < 2497 + } + } + + /// Safe version of `hash_ptr_unsafe` that doesn't hit a stack overflow by + /// precomputing the pointers that need to be hashed in order to hash the + /// provided `ptr` + pub fn hash_ptr(&self, ptr: &RawPtr) -> FWrap { + if self.is_below_safe_threshold() { + // just run `hash_ptr_unsafe` for extra speed when the dehydrated + // queue is small enough + return self.hash_ptr_unsafe(ptr); + } + let mut ptrs: IndexSet<&RawPtr> = IndexSet::default(); + let mut stack = vec![ptr]; + macro_rules! feed_loop { + ($x:expr) => { + if $x.is_hash() { + if self.z_cache.get($x).is_none() { + if ptrs.insert($x) { + stack.push($x); + } + } + } + }; + } + while let Some(ptr) = stack.pop() { + match ptr { + RawPtr::Atom(..) => (), + RawPtr::Hash3(idx) => { + let ptrs = self.expect_3_ptrs(*idx); + for ptr in ptrs { + feed_loop!(ptr) + } + } + RawPtr::Hash4(idx) => { + let ptrs = self.expect_4_ptrs(*idx); + for ptr in ptrs { + feed_loop!(ptr) + } + } + RawPtr::Hash6(idx) => { + let ptrs = self.expect_6_ptrs(*idx); + for ptr in ptrs { + feed_loop!(ptr) + } + } + RawPtr::Hash8(idx) => { + let ptrs = self.expect_8_ptrs(*idx); + for ptr in ptrs { + feed_loop!(ptr) + } + } + } + } + ptrs.reverse(); + self.hydrate_z_cache_with_ptrs(&ptrs.into_iter().collect::>()); + // Now it's okay to call `hash_ptr_unsafe` + self.hash_ptr_unsafe(ptr) + } + + /// Constructs a vector of scalars that correspond to tags and hashes computed + /// from a slice of `Ptr`s turned into `ZPtr`s + pub fn to_scalar_vector(&self, ptrs: &[RawPtr]) -> Vec { + ptrs.iter().map(|ptr| self.hash_ptr(ptr).0).collect() + } + + /// Equality of the content-addressed versions of two pointers + #[inline] + pub fn ptr_eq(&self, a: &RawPtr, b: &RawPtr) -> bool { + self.hash_ptr(a) == self.hash_ptr(b) + } + + /// Attempts to recover the `Ptr` that corresponds to `z_ptr` from + /// `inverse_z_cache`. If the mapping is not there, returns an atom pointer + /// with the same tag and value + #[inline] + pub fn to_ptr(&self, z: &FWrap) -> RawPtr { + self.inverse_z_cache + .get(z) + .cloned() + .unwrap_or_else(|| self.opaque(*z)) + } +} + +/* +impl Ptr { + pub fn fmt_to_string(&self, store: &Store, state: &State) -> String { + match self.tag() { + Tag::Expr(t) => match t { + Nil => { + if let Some(sym) = store.fetch_symbol(self) { + state.fmt_to_string(&sym.into()) + } else { + "".into() + } + } + Sym => { + if let Some(sym) = store.fetch_sym(self) { + state.fmt_to_string(&sym.into()) + } else { + "".into() + } + } + Key => { + if let Some(key) = store.fetch_key(self) { + state.fmt_to_string(&key.into()) + } else { + "".into() + } + } + Str => { + if let Some(str) = store.fetch_string(self) { + format!("\"{str}\"") + } else { + "".into() + } + } + Char => { + if let Some(c) = self + .get_atom() + .map(|idx| store.expect_f(idx)) + .and_then(F::to_char) + { + format!("\'{c}\'") + } else { + "".into() + } + } + Cons => { + if let Some((list, non_nil)) = store.fetch_list(self) { + let list = list + .iter() + .map(|p| p.fmt_to_string(store, state)) + .collect::>(); + if let Some(non_nil) = non_nil { + format!( + "({} . {})", + list.join(" "), + non_nil.fmt_to_string(store, state) + ) + } else { + format!("({})", list.join(" ")) + } + } else { + "".into() + } + } + Num => { + if let Some(f) = self.get_atom().map(|idx| store.expect_f(idx)) { + if let Some(u) = f.to_u64() { + u.to_string() + } else { + format!("0x{}", f.hex_digits()) + } + } else { + "".into() + } + } + U64 => { + if let Some(u) = self + .get_atom() + .map(|idx| store.expect_f(idx)) + .and_then(F::to_u64) + { + format!("{u}u64") + } else { + "".into() + } + } + Fun => match self.get_index4() { + None => "".into(), + Some(idx) => { + if let Some((vars, body, ..)) = store.fetch_4_ptrs(idx) { + match vars.tag() { + Tag::Expr(Nil) => { + format!("", body.fmt_to_string(store, state)) + } + Tag::Expr(Cons) => { + format!( + "", + vars.fmt_to_string(store, state), + body.fmt_to_string(store, state) + ) + } + _ => "".into(), + } + } else { + "".into() + } + } + }, + Thunk => match self.get_index2() { + None => "".into(), + Some(idx) => { + if let Some((val, cont)) = store.fetch_2_ptrs(idx) { + format!( + "Thunk{{ value: {} => cont: {} }}", + val.fmt_to_string(store, state), + cont.fmt_to_string(store, state) + ) + } else { + "".into() + } + } + }, + Comm => match self.get_atom() { + Some(idx) => { + let f = store.expect_f(idx); + if store.comms.get(&FWrap(*f)).is_some() { + format!("(comm 0x{})", f.hex_digits()) + } else { + format!("", f.hex_digits()) + } + } + None => "".into(), + }, + Cproc => match self.get_index2() { + None => "".into(), + Some(idx) => { + if let Some((cproc_name, args)) = store.fetch_2_ptrs(idx) { + format!( + "", + cproc_name.fmt_to_string(store, state), + args.fmt_to_string(store, state) + ) + } else { + "".into() + } + } + }, + }, + Tag::Cont(t) => match t { + Outermost => "Outermost".into(), + Dummy => "Dummy".into(), + ContTag::Error => "Error".into(), + Terminal => "Terminal".into(), + Call0 => self.fmt_cont2_to_string("Call0", "saved_env", store, state), + Call => { + self.fmt_cont3_to_string("Call", ("unevaled_arg", "saved_env"), store, state) + } + Call2 => self.fmt_cont3_to_string("Call2", ("function", "saved_env"), store, state), + Tail => self.fmt_cont2_to_string("Tail", "saved_env", store, state), + Lookup => self.fmt_cont2_to_string("Lookup", "saved_env", store, state), + Unop => self.fmt_cont2_to_string("Unop", "saved_env", store, state), + Binop => self.fmt_cont4_to_string( + "Binop", + ("operator", "saved_env", "unevaled_args"), + store, + state, + ), + Binop2 => { + self.fmt_cont3_to_string("Binop2", ("operator", "evaled_arg"), store, state) + } + If => self.fmt_cont2_to_string("If", "unevaled_args", store, state), + Let => self.fmt_cont4_to_string("Let", ("var", "saved_env", "body"), store, state), + LetRec => { + self.fmt_cont4_to_string("LetRec", ("var", "saved_env", "body"), store, state) + } + Emit => "Emit ".into(), + ContTag::Cproc => self.fmt_cont4_to_string( + "Cproc", + ("name", "unevaled_args", "evaled_args"), + store, + state, + ), + }, + Tag::Op1(op) => op.to_string(), + Tag::Op2(op) => op.to_string(), + } + } + + fn fmt_cont2_to_string( + &self, + name: &str, + field: &str, + store: &Store, + state: &State, + ) -> String { + match self.get_index4() { + None => format!(""), + Some(idx) => { + if let Some((a, cont, ..)) = store.fetch_4_ptrs(idx) { + format!( + "{name}{{ {field}: {}, continuation: {} }}", + a.fmt_to_string(store, state), + cont.fmt_to_string(store, state) + ) + } else { + format!("") + } + } + } + } + + fn fmt_cont3_to_string( + &self, + name: &str, + fields: (&str, &str), + store: &Store, + state: &State, + ) -> String { + match self.get_index4() { + None => format!(""), + Some(idx) => { + if let Some((a, b, cont, _)) = store.fetch_4_ptrs(idx) { + let (fa, fb) = fields; + format!( + "{name}{{ {fa}: {}, {fb}: {}, continuation: {} }}", + a.fmt_to_string(store, state), + b.fmt_to_string(store, state), + cont.fmt_to_string(store, state) + ) + } else { + format!("") + } + } + } + } + + fn fmt_cont4_to_string( + &self, + name: &str, + fields: (&str, &str, &str), + store: &Store, + state: &State, + ) -> String { + match self.get_index4() { + None => format!(""), + Some(idx) => { + if let Some((a, b, c, cont)) = store.fetch_4_ptrs(idx) { + let (fa, fb, fc) = fields; + format!( + "{name}{{ {fa}: {}, {fb}: {}, {fc}: {}, continuation: {} }}", + a.fmt_to_string(store, state), + b.fmt_to_string(store, state), + c.fmt_to_string(store, state), + cont.fmt_to_string(store, state) + ) + } else { + format!("") + } + } + } + } +} +*/ From cd31d0aa652d7e9507722aa7136bda4033062786 Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Thu, 28 Dec 2023 14:47:02 -0300 Subject: [PATCH 02/12] const generic store functions --- src/lem/raw_store.rs | 279 +++++++++++++++++++------------------------ 1 file changed, 125 insertions(+), 154 deletions(-) diff --git a/src/lem/raw_store.rs b/src/lem/raw_store.rs index 834cfcf840..d0e694f1a1 100644 --- a/src/lem/raw_store.rs +++ b/src/lem/raw_store.rs @@ -151,39 +151,29 @@ impl Store { self.f_elts.insert_probe(Box::new(FWrap(f))) } - /// Creates an atom `Ptr` which points to a cached element of the finite + /// Creates an atom `RawPtr` which points to a cached element of the finite /// field `F` pub fn intern_atom(&self, f: F) -> RawPtr { let (idx, _) = self.intern_f(f); RawPtr::Atom(idx) } - /// Creates a `Ptr` that's a parent of 3 children - pub fn intern_3_ptrs(&self, ptrs: [RawPtr; 3]) -> RawPtr { - let (idx, inserted) = self.hash3.insert_probe(Box::new(ptrs)); - let ptr = RawPtr::Hash3(idx); - if inserted { - // this is for `hydrate_z_cache` - self.dehydrated.load().push(Box::new(ptr)); + /// Creates a `RawPtr` that's a parent of `ARITY` children + pub fn intern_raw_ptrs(&self, ptrs: [RawPtr; ARITY]) -> RawPtr { + macro_rules! intern { + ($Hash:ident, $hash:ident, $n:expr) => {{ + let ptrs: &[RawPtr; $n] = unsafe { std::mem::transmute(&ptrs) }; + let (idx, inserted) = self.$hash.insert_probe(Box::new(*ptrs)); + (RawPtr::$Hash(idx), inserted) + }}; } - ptr - } - - /// Similar to `intern_3_ptrs` but doesn't add the resulting pointer to - /// `dehydrated`. This function is used when converting a `ZStore` to a - /// `Store`. - pub fn intern_3_ptrs_hydrated(&self, ptrs: [RawPtr; 3], z: FWrap) -> RawPtr { - let (idx, _) = self.hash3.insert_probe(Box::new(ptrs)); - let ptr = RawPtr::Hash3(idx); - self.z_cache.insert(ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(ptr)); - ptr - } - - /// Creates a `Ptr` that's a parent of 4 children - pub fn intern_4_ptrs(&self, ptrs: [RawPtr; 4]) -> RawPtr { - let (idx, inserted) = self.hash4.insert_probe(Box::new(ptrs)); - let ptr = RawPtr::Hash4(idx); + let (ptr, inserted) = match ARITY { + 3 => intern!(Hash3, hash3, 3), + 4 => intern!(Hash4, hash4, 4), + 6 => intern!(Hash6, hash6, 6), + 8 => intern!(Hash8, hash8, 8), + _ => unimplemented!(), + }; if inserted { // this is for `hydrate_z_cache` self.dehydrated.load().push(Box::new(ptr)); @@ -191,59 +181,80 @@ impl Store { ptr } - /// Similar to `intern_4_ptrs` but doesn't add the resulting pointer to + /// Similar to `intern_raw_ptrs` but doesn't add the resulting pointer to /// `dehydrated`. This function is used when converting a `ZStore` to a /// `Store`. - pub fn intern_4_ptrs_hydrated(&self, ptrs: [RawPtr; 4], z: FWrap) -> RawPtr { - let (idx, _) = self.hash4.insert_probe(Box::new(ptrs)); - let ptr = RawPtr::Hash4(idx); - self.z_cache.insert(ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(ptr)); - ptr - } - - /// Creates a `Ptr` that's a parent of 6 children - pub fn intern_6_ptrs(&self, ptrs: [RawPtr; 6]) -> RawPtr { - let (idx, inserted) = self.hash6.insert_probe(Box::new(ptrs)); - let ptr = RawPtr::Hash6(idx); - if inserted { - // this is for `hydrate_z_cache` - self.dehydrated.load().push(Box::new(ptr)); + pub fn intern_raw_ptrs_hydrated( + &self, + ptrs: [RawPtr; ARITY], + z: FWrap, + ) -> RawPtr { + macro_rules! intern { + ($Hash:ident, $hash:ident, $n:expr) => {{ + let ptrs: &[RawPtr; $n] = unsafe { std::mem::transmute(&ptrs) }; + let (idx, _) = self.$hash.insert_probe(Box::new(*ptrs)); + RawPtr::$Hash(idx) + }}; } - ptr - } - - /// Similar to `intern_6_ptrs` but doesn't add the resulting pointer to - /// `dehydrated`. This function is used when converting a `ZStore` to a - /// `Store`. - pub fn intern_6_ptrs_hydrated(&self, ptrs: [RawPtr; 6], z: FWrap) -> RawPtr { - let (idx, _) = self.hash6.insert_probe(Box::new(ptrs)); - let ptr = RawPtr::Hash6(idx); + let ptr = match ARITY { + 3 => intern!(Hash3, hash3, 3), + 4 => intern!(Hash4, hash4, 4), + 6 => intern!(Hash6, hash6, 6), + 8 => intern!(Hash8, hash8, 8), + _ => unimplemented!(), + }; self.z_cache.insert(ptr, Box::new(z)); self.inverse_z_cache.insert(z, Box::new(ptr)); ptr } - /// Creates a `Ptr` that's a parent of 8 children - pub fn intern_8_ptrs(&self, ptrs: [RawPtr; 8]) -> RawPtr { - let (idx, inserted) = self.hash8.insert_probe(Box::new(ptrs)); - let ptr = RawPtr::Hash8(idx); - if inserted { - // this is for `hydrate_z_cache` - self.dehydrated.load().push(Box::new(ptr)); + /// Creates a `Ptr` that's a parent of `ARITY` children + pub fn intern_ptrs(&self, tag: Tag, ptrs: [Ptr; ARITY]) -> Ptr { + macro_rules! intern { + ($n:expr) => {{ + let mut raw_ptrs = [self.zero(); $n * 2]; + for i in 0..$n { + raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); + raw_ptrs[2 * i + 1] = *ptrs[i].pay(); + } + self.intern_raw_ptrs::<{ $n * 2 }>(raw_ptrs) + }}; } - ptr + let pay = match ARITY { + 2 => intern!(2), + 3 => intern!(3), + 4 => intern!(4), + _ => unimplemented!(), + }; + Ptr::new(tag, pay) } - /// Similar to `intern_8_ptrs` but doesn't add the resulting pointer to + /// Similar to `intern_ptrs` but doesn't add the resulting pointer to /// `dehydrated`. This function is used when converting a `ZStore` to a /// `Store`. - pub fn intern_8_ptrs_hydrated(&self, ptrs: [RawPtr; 8], z: FWrap) -> RawPtr { - let (idx, _) = self.hash8.insert_probe(Box::new(ptrs)); - let ptr = RawPtr::Hash8(idx); - self.z_cache.insert(ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(ptr)); - ptr + pub fn intern_ptrs_hydrated( + &self, + tag: Tag, + ptrs: [Ptr; ARITY], + z: FWrap, + ) -> Ptr { + macro_rules! intern { + ($n:expr) => {{ + let mut raw_ptrs = [self.zero(); $n * 2]; + for i in 0..$n { + raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); + raw_ptrs[2 * i + 1] = *ptrs[i].pay(); + } + self.intern_raw_ptrs_hydrated::<{ $n * 2 }>(raw_ptrs, z) + }}; + } + let pay = match ARITY { + 2 => intern!(2), + 3 => intern!(3), + 4 => intern!(4), + _ => unimplemented!(), + }; + Ptr::new(tag, pay) } #[inline] @@ -252,23 +263,21 @@ impl Store { } #[inline] - pub fn fetch_3_ptrs(&self, idx: usize) -> Option<&[RawPtr; 3]> { - self.hash3.get_index(idx) - } - - #[inline] - pub fn fetch_4_ptrs(&self, idx: usize) -> Option<&[RawPtr; 4]> { - self.hash4.get_index(idx) - } - - #[inline] - pub fn fetch_6_ptrs(&self, idx: usize) -> Option<&[RawPtr; 6]> { - self.hash6.get_index(idx) - } - - #[inline] - pub fn fetch_8_ptrs(&self, idx: usize) -> Option<&[RawPtr; 8]> { - self.hash8.get_index(idx) + pub fn fetch_raw_ptrs(&self, idx: usize) -> Option<&[RawPtr; ARITY]> { + macro_rules! fetch { + ($hash:ident, $n:expr) => {{ + let ptrs = self.$hash.get_index(idx)?; + let ptrs: &[RawPtr; ARITY] = unsafe { std::mem::transmute(&ptrs) }; + Some(ptrs) + }}; + } + match ARITY { + 3 => fetch!(hash3, 3), + 4 => fetch!(hash4, 4), + 6 => fetch!(hash6, 6), + 8 => fetch!(hash8, 8), + _ => unimplemented!(), + } } #[inline] @@ -277,23 +286,8 @@ impl Store { } #[inline] - pub fn expect_3_ptrs(&self, idx: usize) -> &[RawPtr; 3] { - self.fetch_3_ptrs(idx).expect("Index missing from store") - } - - #[inline] - pub fn expect_4_ptrs(&self, idx: usize) -> &[RawPtr; 4] { - self.fetch_4_ptrs(idx).expect("Index missing from store") - } - - #[inline] - pub fn expect_6_ptrs(&self, idx: usize) -> &[RawPtr; 6] { - self.fetch_6_ptrs(idx).expect("Index missing from store") - } - - #[inline] - pub fn expect_8_ptrs(&self, idx: usize) -> &[RawPtr; 8] { - self.fetch_8_ptrs(idx).expect("Index missing from store") + pub fn expect_raw_ptrs(&self, idx: usize) -> &[RawPtr; ARITY] { + self.fetch_raw_ptrs::(idx).expect("Index missing from store") } #[inline] @@ -351,8 +345,8 @@ impl Store { } #[inline] - pub fn dummy(&self) -> RawPtr { - self.zero() + pub fn dummy(&self) -> Ptr { + Ptr::new(Tag::Expr(Nil), self.zero()) } /// Creates an atom pointer from a `ZPtr`, with its hash. Hashing @@ -366,16 +360,11 @@ impl Store { if let Some(ptr) = self.string_ptr_cache.get(s) { *ptr } else { - let pay = s.chars().rev().fold(self.zero(), |acc, c| { - let ptrs = [ - self.tag(Tag::Expr(Char)), - *self.char(c).pay(), - self.tag(Tag::Expr(Str)), - acc, - ]; - self.intern_4_ptrs(ptrs) + let nil_str = Ptr::new(Tag::Expr(Str), self.zero()); + let ptr = s.chars().rev().fold(nil_str, |acc, c| { + let ptrs = [self.char(c), acc]; + self.intern_ptrs::<2>(Tag::Expr(Str), ptrs) }); - let ptr = Ptr::new(Tag::Expr(Str), pay); self.string_ptr_cache.insert(s.to_string(), Box::new(ptr)); self.ptr_string_cache.insert(ptr, s.to_string()); ptr @@ -402,7 +391,7 @@ impl Store { } } RawPtr::Hash4(idx) => { - let [car_tag, car, cdr_tag, cdr] = self.fetch_4_ptrs(idx)?; + let [car_tag, car, cdr_tag, cdr] = self.fetch_raw_ptrs(idx)?; assert_eq!(*car_tag, self.tag(Tag::Expr(Char))); assert_eq!(*cdr_tag, self.tag(Tag::Expr(Str))); match car { @@ -420,15 +409,11 @@ impl Store { } } - pub fn intern_symbol_path(&self, path: &[String]) -> RawPtr { - path.iter().fold(self.zero(), |acc, s| { - let ptrs = [ - self.tag(Tag::Expr(Str)), - *self.intern_string(s).pay(), - self.tag(Tag::Expr(Sym)), - acc, - ]; - self.intern_4_ptrs(ptrs) + pub fn intern_symbol_path(&self, path: &[String]) -> Ptr { + let zero_sym = Ptr::new(Tag::Expr(Sym), self.zero()); + path.iter().fold(zero_sym, |acc, s| { + let ptrs = [self.intern_string(s), acc]; + self.intern_ptrs::<2>(Tag::Expr(Sym), ptrs) }) } @@ -438,11 +423,11 @@ impl Store { } else { let path_ptr = self.intern_symbol_path(sym.path()); let sym_ptr = if sym == &lurk_sym("nil") { - Ptr::new(Tag::Expr(Nil), path_ptr) + Ptr::new(Tag::Expr(Nil), *path_ptr.pay()) } else if sym.is_keyword() { - Ptr::new(Tag::Expr(Key), path_ptr) + Ptr::new(Tag::Expr(Key), *path_ptr.pay()) } else { - Ptr::new(Tag::Expr(Sym), path_ptr) + path_ptr }; self.symbol_ptr_cache.insert(sym.clone(), Box::new(sym_ptr)); self.ptr_symbol_cache.insert(sym_ptr, Box::new(sym.clone())); @@ -454,7 +439,7 @@ impl Store { fn fetch_symbol_path(&self, mut idx: usize) -> Option> { let mut path = vec![]; loop { - let [car_tag, car, cdr_tag, cdr] = self.fetch_4_ptrs(idx)?; + let [car_tag, car, cdr_tag, cdr] = self.fetch_raw_ptrs(idx)?; assert_eq!(*car_tag, self.tag(Tag::Expr(Str))); assert_eq!(*cdr_tag, self.tag(Tag::Expr(Sym))); let string = self.fetch_string(&Ptr::new(Tag::Expr(Str), *car))?; @@ -582,28 +567,14 @@ impl Store { #[inline] pub fn cons(&self, car: Ptr, cdr: Ptr) -> Ptr { - let ptrs = [ - self.tag(*car.tag()), - *car.pay(), - self.tag(*cdr.tag()), - *cdr.pay(), - ]; - Ptr::new(Tag::Expr(Cons), self.intern_4_ptrs(ptrs)) + let ptrs = [car, cdr]; + self.intern_ptrs::<2>(Tag::Expr(Cons), ptrs) } #[inline] pub fn intern_fun(&self, arg: Ptr, body: Ptr, env: Ptr) -> Ptr { - let ptrs = [ - self.tag(*arg.tag()), - *arg.pay(), - self.tag(*body.tag()), - *body.pay(), - self.tag(*env.tag()), - *env.pay(), - self.tag(Tag::Expr(Nil)), - self.dummy(), - ]; - Ptr::new(Tag::Expr(Fun), self.intern_8_ptrs(ptrs)) + let ptrs = [arg, body, env, self.dummy()]; + self.intern_ptrs::<4>(Tag::Expr(Fun), ptrs) } #[inline] @@ -631,7 +602,7 @@ impl Store { let Some(idx) = ptr.pay().get_hash4() else { bail!("malformed cons pointer") }; - match self.fetch_4_ptrs(idx) { + match self.fetch_raw_ptrs(idx) { Some([car_tag, car, cdr_tag, cdr]) => { let car_ptr = self.raw_to_ptr(car_tag, car).context("Not a pointer")?; let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr).context("Not a pointer")?; @@ -648,7 +619,7 @@ impl Store { let Some(idx) = ptr.pay().get_hash4() else { bail!("malformed str pointer") }; - match self.fetch_4_ptrs(idx) { + match self.fetch_raw_ptrs(idx) { Some([car_tag, car, cdr_tag, cdr]) => { let car_ptr = self.raw_to_ptr(car_tag, car).context("Not a pointer")?; let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr).context("Not a pointer")?; @@ -696,7 +667,7 @@ impl Store { (Tag::Expr(Cons), RawPtr::Hash4(mut idx)) => { let mut list = vec![]; let mut last = None; - while let Some([car_tag, car, cdr_tag, cdr]) = self.fetch_4_ptrs(idx) { + while let Some([car_tag, car, cdr_tag, cdr]) = self.fetch_raw_ptrs(idx) { let car_ptr = self.raw_to_ptr(car_tag, car)?; let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr)?; list.push(car_ptr); @@ -790,7 +761,7 @@ impl Store { if let Some(z) = self.z_cache.get(ptr) { *z } else { - let children_ptrs = self.expect_3_ptrs(*idx); + let children_ptrs = self.expect_raw_ptrs::<3>(*idx); let mut children_zs = [F::ZERO; 3]; for (idx, child_ptr) in children_ptrs.iter().enumerate() { children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; @@ -805,7 +776,7 @@ impl Store { if let Some(z) = self.z_cache.get(ptr) { *z } else { - let children_ptrs = self.expect_4_ptrs(*idx); + let children_ptrs = self.expect_raw_ptrs::<4>(*idx); let mut children_zs = [F::ZERO; 4]; for (idx, child_ptr) in children_ptrs.iter().enumerate() { children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; @@ -820,7 +791,7 @@ impl Store { if let Some(z) = self.z_cache.get(ptr) { *z } else { - let children_ptrs = self.expect_6_ptrs(*idx); + let children_ptrs = self.expect_raw_ptrs::<6>(*idx); let mut children_zs = [F::ZERO; 6]; for (idx, child_ptr) in children_ptrs.iter().enumerate() { children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; @@ -835,7 +806,7 @@ impl Store { if let Some(z) = self.z_cache.get(ptr) { *z } else { - let children_ptrs = self.expect_8_ptrs(*idx); + let children_ptrs = self.expect_raw_ptrs::<8>(*idx); let mut children_zs = [F::ZERO; 8]; for (idx, child_ptr) in children_ptrs.iter().enumerate() { children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; @@ -909,25 +880,25 @@ impl Store { match ptr { RawPtr::Atom(..) => (), RawPtr::Hash3(idx) => { - let ptrs = self.expect_3_ptrs(*idx); + let ptrs = self.expect_raw_ptrs::<3>(*idx); for ptr in ptrs { feed_loop!(ptr) } } RawPtr::Hash4(idx) => { - let ptrs = self.expect_4_ptrs(*idx); + let ptrs = self.expect_raw_ptrs::<4>(*idx); for ptr in ptrs { feed_loop!(ptr) } } RawPtr::Hash6(idx) => { - let ptrs = self.expect_6_ptrs(*idx); + let ptrs = self.expect_raw_ptrs::<6>(*idx); for ptr in ptrs { feed_loop!(ptr) } } RawPtr::Hash8(idx) => { - let ptrs = self.expect_8_ptrs(*idx); + let ptrs = self.expect_raw_ptrs::<8>(*idx); for ptr in ptrs { feed_loop!(ptr) } From 237fd7246910ff84bef0819b9d0d39fd375e1004 Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Fri, 29 Dec 2023 14:18:50 -0300 Subject: [PATCH 03/12] Finished raw store --- src/lem/raw_pointers.rs | 66 ++++++++++- src/lem/raw_store.rs | 257 +++++++++++++++++++++++++++++----------- 2 files changed, 255 insertions(+), 68 deletions(-) diff --git a/src/lem/raw_pointers.rs b/src/lem/raw_pointers.rs index ab3c440f62..f89217745b 100644 --- a/src/lem/raw_pointers.rs +++ b/src/lem/raw_pointers.rs @@ -1,7 +1,9 @@ -use super::Tag; -use crate::tag::ExprTag::{Cons, Fun, Nil, Num, Str, Sym}; use serde::{Deserialize, Serialize}; +use crate::tag::ExprTag::{Cons, Fun, Nil, Num, Str, Sym}; + +use super::Tag; + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] pub enum RawPtr { Atom(usize), @@ -61,6 +63,17 @@ impl RawPtr { } } +/// `Ptr` is the main piece of data LEMs operate on. We can think of a pointer +/// as a building block for trees that represent Lurk data. A pointer can be a +/// atom that contains data encoded as an element of a `LurkField` or it can have +/// children. For performance, the children of a pointer are stored on an +/// `IndexSet` and the resulding index is carried by the pointer itself. +/// +/// A pointer also has a tag, which says what kind of data it encodes. On +/// previous implementations, the tag would be used to infer the number of +/// children a pointer has. However, LEMs require extra flexibility because LEM +/// hashing operations can plug any tag to the resulting pointer. Thus, the +/// number of children have to be made explicit as the `Ptr` enum. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] pub struct Ptr { tag: Tag, @@ -124,4 +137,53 @@ impl Ptr { pub fn cast(self, tag: Tag) -> Self { Ptr { tag, pay: self.pay } } + + #[inline] + pub fn get_atom(&self) -> Option { + self.pay().get_atom() + } + + #[inline] + pub fn get_index2(&self) -> Option { + self.pay().get_hash4() + } + + #[inline] + pub fn get_index3(&self) -> Option { + self.pay().get_hash6() + } + + #[inline] + pub fn get_index4(&self) -> Option { + self.pay().get_hash8() + } + + #[inline] + pub fn atom(tag: Tag, idx: usize) -> Ptr { + Ptr { + tag, + pay: RawPtr::Atom(idx), + } + } +} + +/// A `ZPtr` is the result of "hydrating" a `Ptr`. This process is better +/// explained in the store but, in short, we want to know the Poseidon hash of +/// the children of a `Ptr`. +/// +/// `ZPtr`s are used mainly for proofs, but they're also useful when we want +/// to content-address a store. +/// +/// An important note is that computing the respective `ZPtr` of a `Ptr` can be +/// expensive because of the Poseidon hashes. That's why we operate on `Ptr`s +/// when interpreting LEMs and delay the need for `ZPtr`s as much as possible. +pub type ZPtr = crate::z_data::z_ptr::ZPtr; + +/* +impl ZPtr { + #[inline] + pub fn dummy() -> Self { + Self(Tag::Expr(Nil), F::ZERO) + } } +*/ diff --git a/src/lem/raw_store.rs b/src/lem/raw_store.rs index d0e694f1a1..6c9d906d05 100644 --- a/src/lem/raw_store.rs +++ b/src/lem/raw_store.rs @@ -27,7 +27,7 @@ use crate::{ tag::Tag as TagTrait, }; -use super::raw_pointers::{Ptr, RawPtr}; +use super::raw_pointers::{Ptr, RawPtr, ZPtr}; #[derive(Debug)] pub struct Store { @@ -153,21 +153,25 @@ impl Store { /// Creates an atom `RawPtr` which points to a cached element of the finite /// field `F` - pub fn intern_atom(&self, f: F) -> RawPtr { + pub fn intern_raw_atom(&self, f: F) -> RawPtr { let (idx, _) = self.intern_f(f); RawPtr::Atom(idx) } - /// Creates a `RawPtr` that's a parent of `ARITY` children - pub fn intern_raw_ptrs(&self, ptrs: [RawPtr; ARITY]) -> RawPtr { + pub fn intern_atom(&self, tag: Tag, f: F) -> Ptr { + Ptr::new(tag, self.intern_raw_atom(f)) + } + + /// Creates a `RawPtr` that's a parent of `N` children + pub fn intern_raw_ptrs(&self, ptrs: [RawPtr; N]) -> RawPtr { macro_rules! intern { ($Hash:ident, $hash:ident, $n:expr) => {{ - let ptrs: &[RawPtr; $n] = unsafe { std::mem::transmute(&ptrs) }; + let ptrs = unsafe { std::mem::transmute::<&[RawPtr; N], &[RawPtr; $n]>(&ptrs) }; let (idx, inserted) = self.$hash.insert_probe(Box::new(*ptrs)); (RawPtr::$Hash(idx), inserted) }}; } - let (ptr, inserted) = match ARITY { + let (ptr, inserted) = match N { 3 => intern!(Hash3, hash3, 3), 4 => intern!(Hash4, hash4, 4), 6 => intern!(Hash6, hash6, 6), @@ -184,19 +188,19 @@ impl Store { /// Similar to `intern_raw_ptrs` but doesn't add the resulting pointer to /// `dehydrated`. This function is used when converting a `ZStore` to a /// `Store`. - pub fn intern_raw_ptrs_hydrated( + pub fn intern_raw_ptrs_hydrated( &self, - ptrs: [RawPtr; ARITY], + ptrs: [RawPtr; N], z: FWrap, ) -> RawPtr { macro_rules! intern { ($Hash:ident, $hash:ident, $n:expr) => {{ - let ptrs: &[RawPtr; $n] = unsafe { std::mem::transmute(&ptrs) }; + let ptrs = unsafe { std::mem::transmute::<&[RawPtr; N], &[RawPtr; $n]>(&ptrs) }; let (idx, _) = self.$hash.insert_probe(Box::new(*ptrs)); RawPtr::$Hash(idx) }}; } - let ptr = match ARITY { + let ptr = match N { 3 => intern!(Hash3, hash3, 3), 4 => intern!(Hash4, hash4, 4), 6 => intern!(Hash6, hash6, 6), @@ -208,11 +212,11 @@ impl Store { ptr } - /// Creates a `Ptr` that's a parent of `ARITY` children - pub fn intern_ptrs(&self, tag: Tag, ptrs: [Ptr; ARITY]) -> Ptr { + /// Creates a `Ptr` that's a parent of `N` children + pub fn intern_ptrs(&self, tag: Tag, ptrs: [Ptr; N]) -> Ptr { macro_rules! intern { ($n:expr) => {{ - let mut raw_ptrs = [self.zero(); $n * 2]; + let mut raw_ptrs = [self.raw_zero(); $n * 2]; for i in 0..$n { raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); raw_ptrs[2 * i + 1] = *ptrs[i].pay(); @@ -220,7 +224,7 @@ impl Store { self.intern_raw_ptrs::<{ $n * 2 }>(raw_ptrs) }}; } - let pay = match ARITY { + let pay = match N { 2 => intern!(2), 3 => intern!(3), 4 => intern!(4), @@ -232,15 +236,15 @@ impl Store { /// Similar to `intern_ptrs` but doesn't add the resulting pointer to /// `dehydrated`. This function is used when converting a `ZStore` to a /// `Store`. - pub fn intern_ptrs_hydrated( + pub fn intern_ptrs_hydrated( &self, tag: Tag, - ptrs: [Ptr; ARITY], + ptrs: [Ptr; N], z: FWrap, ) -> Ptr { macro_rules! intern { ($n:expr) => {{ - let mut raw_ptrs = [self.zero(); $n * 2]; + let mut raw_ptrs = [self.raw_zero(); $n * 2]; for i in 0..$n { raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); raw_ptrs[2 * i + 1] = *ptrs[i].pay(); @@ -248,7 +252,7 @@ impl Store { self.intern_raw_ptrs_hydrated::<{ $n * 2 }>(raw_ptrs, z) }}; } - let pay = match ARITY { + let pay = match N { 2 => intern!(2), 3 => intern!(3), 4 => intern!(4), @@ -263,15 +267,15 @@ impl Store { } #[inline] - pub fn fetch_raw_ptrs(&self, idx: usize) -> Option<&[RawPtr; ARITY]> { + pub fn fetch_raw_ptrs(&self, idx: usize) -> Option<&[RawPtr; N]> { macro_rules! fetch { ($hash:ident, $n:expr) => {{ let ptrs = self.$hash.get_index(idx)?; - let ptrs: &[RawPtr; ARITY] = unsafe { std::mem::transmute(&ptrs) }; + let ptrs = unsafe { std::mem::transmute::<&[RawPtr; $n], &[RawPtr; N]>(ptrs) }; Some(ptrs) }}; } - match ARITY { + match N { 3 => fetch!(hash3, 3), 4 => fetch!(hash4, 4), 6 => fetch!(hash6, 6), @@ -280,19 +284,101 @@ impl Store { } } + #[inline] + pub fn fetch_ptrs(&self, idx: usize) -> Option<[Ptr; P]> { + assert_eq!(P * 2, N); + let raw_ptrs = self + .fetch_raw_ptrs::(idx) + .expect("Index missing from store"); + let mut ptrs = [self.dummy(); P]; + for i in 0..P { + let tag = self.fetch_tag(&raw_ptrs[2 * i])?; + ptrs[i] = Ptr::new(tag, raw_ptrs[2 * i + 1]) + } + Some(ptrs) + } + #[inline] pub fn expect_f(&self, idx: usize) -> &F { self.fetch_f(idx).expect("Index missing from f_elts") } #[inline] - pub fn expect_raw_ptrs(&self, idx: usize) -> &[RawPtr; ARITY] { - self.fetch_raw_ptrs::(idx).expect("Index missing from store") + pub fn expect_raw_ptrs(&self, idx: usize) -> &[RawPtr; N] { + self.fetch_raw_ptrs::(idx) + .expect("Index missing from store") + } + + #[inline] + pub fn expect_ptrs(&self, idx: usize) -> [Ptr; P] { + self.fetch_ptrs::(idx) + .expect("Index missing from store") + } + + // TODO remove these functions + pub fn intern_atom_hydrated(&self, tag: Tag, f: F, _: ZPtr) -> Ptr { + Ptr::new(tag, self.intern_raw_atom(f)) + } + #[inline] + pub fn intern_2_ptrs(&self, tag: Tag, a: Ptr, b: Ptr) -> Ptr { + self.intern_ptrs::<2>(tag, [a, b]) + } + #[inline] + pub fn intern_3_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr) -> Ptr { + self.intern_ptrs::<3>(tag, [a, b, c]) + } + #[inline] + pub fn intern_4_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, d: Ptr) -> Ptr { + self.intern_ptrs::<4>(tag, [a, b, c, d]) + } + #[inline] + pub fn intern_2_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, z: ZPtr) -> Ptr { + self.intern_ptrs_hydrated::<2>(tag, [a, b], FWrap(*z.value())) + } + #[inline] + pub fn intern_3_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, z: ZPtr) -> Ptr { + self.intern_ptrs_hydrated::<3>(tag, [a, b, c], FWrap(*z.value())) + } + #[inline] + pub fn intern_4_ptrs_hydrated( + &self, + tag: Tag, + a: Ptr, + b: Ptr, + c: Ptr, + d: Ptr, + z: ZPtr, + ) -> Ptr { + self.intern_ptrs_hydrated::<4>(tag, [a, b, c, d], FWrap(*z.value())) + } + #[inline] + pub fn fetch_2_ptrs(&self, idx: usize) -> Option<[Ptr; 2]> { + self.fetch_ptrs::<4, 2>(idx) + } + #[inline] + pub fn fetch_3_ptrs(&self, idx: usize) -> Option<[Ptr; 3]> { + self.fetch_ptrs::<6, 3>(idx) + } + #[inline] + pub fn fetch_4_ptrs(&self, idx: usize) -> Option<[Ptr; 4]> { + self.fetch_ptrs::<8, 4>(idx) + } + #[inline] + pub fn expect_2_ptrs(&self, idx: usize) -> [Ptr; 2] { + self.fetch_2_ptrs(idx).expect("Index missing from store") + } + #[inline] + pub fn expect_3_ptrs(&self, idx: usize) -> [Ptr; 3] { + self.fetch_3_ptrs(idx).expect("Index missing from store") + } + #[inline] + pub fn expect_4_ptrs(&self, idx: usize) -> [Ptr; 4] { + self.fetch_4_ptrs(idx).expect("Index missing from store") } #[inline] pub fn tag(&self, tag: Tag) -> RawPtr { - self.intern_atom(tag.to_field()) + self.intern_raw_atom(tag.to_field()) } #[inline] @@ -309,32 +395,37 @@ impl Store { #[inline] pub fn num(&self, f: F) -> Ptr { - Ptr::new(Tag::Expr(Num), self.intern_atom(f)) + self.intern_atom(Tag::Expr(Num), f) } #[inline] pub fn num_u64(&self, u: u64) -> Ptr { - Ptr::new(Tag::Expr(Num), self.intern_atom(F::from_u64(u))) + self.intern_atom(Tag::Expr(Num), F::from_u64(u)) } #[inline] pub fn u64(&self, u: u64) -> Ptr { - Ptr::new(Tag::Expr(U64), self.intern_atom(F::from_u64(u))) + self.intern_atom(Tag::Expr(U64), F::from_u64(u)) } #[inline] pub fn char(&self, c: char) -> Ptr { - Ptr::new(Tag::Expr(Char), self.intern_atom(F::from_char(c))) + self.intern_atom(Tag::Expr(Char), F::from_char(c)) } #[inline] pub fn comm(&self, hash: F) -> Ptr { - Ptr::new(Tag::Expr(Comm), self.intern_atom(hash)) + self.intern_atom(Tag::Expr(Comm), hash) } #[inline] - pub fn zero(&self) -> RawPtr { - self.intern_atom(F::ZERO) + pub fn raw_zero(&self) -> RawPtr { + self.intern_raw_atom(F::ZERO) + } + + #[inline] + pub fn zero(&self, tag: Tag) -> Ptr { + Ptr::new(tag, self.raw_zero()) } pub fn is_zero(&self, ptr: &RawPtr) -> bool { @@ -346,21 +437,21 @@ impl Store { #[inline] pub fn dummy(&self) -> Ptr { - Ptr::new(Tag::Expr(Nil), self.zero()) + Ptr::new(Tag::Expr(Nil), self.raw_zero()) } /// Creates an atom pointer from a `ZPtr`, with its hash. Hashing /// such pointer will result on the same original `ZPtr` #[inline] pub fn opaque(&self, z: FWrap) -> RawPtr { - self.intern_atom(z.0) + self.intern_raw_atom(z.0) } pub fn intern_string(&self, s: &str) -> Ptr { if let Some(ptr) = self.string_ptr_cache.get(s) { *ptr } else { - let nil_str = Ptr::new(Tag::Expr(Str), self.zero()); + let nil_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); let ptr = s.chars().rev().fold(nil_str, |acc, c| { let ptrs = [self.char(c), acc]; self.intern_ptrs::<2>(Tag::Expr(Str), ptrs) @@ -410,7 +501,7 @@ impl Store { } pub fn intern_symbol_path(&self, path: &[String]) -> Ptr { - let zero_sym = Ptr::new(Tag::Expr(Sym), self.zero()); + let zero_sym = Ptr::new(Tag::Expr(Sym), self.raw_zero()); path.iter().fold(zero_sym, |acc, s| { let ptrs = [self.intern_string(s), acc]; self.intern_ptrs::<2>(Tag::Expr(Sym), ptrs) @@ -545,6 +636,15 @@ impl Store { self.comm(self.hide_ptr(secret, payload)) } + pub fn hide_and_return_z_payload(&self, secret: F, payload: Ptr) -> (F, ZPtr) { + let z_ptr = self.hash_ptr(&payload); + let hash = self + .poseidon_cache + .hash3(&[secret, z_ptr.tag_field(), *z_ptr.value()]); + self.add_comm(hash, secret, payload); + (hash, z_ptr) + } + #[inline] pub fn commit(&self, payload: Ptr) -> Ptr { self.hide(F::NON_HIDING_COMMITMENT_SECRET, payload) @@ -559,7 +659,7 @@ impl Store { let hash = self.poseidon_cache.hash3(&[ secret, payload.tag().to_field(), - self.hash_ptr(payload.pay()).0, + self.hash_raw_ptr(payload.pay()).0, ]); self.add_comm(hash, secret, payload); hash @@ -613,7 +713,7 @@ impl Store { } Tag::Expr(Str) => { if self.is_zero(ptr.pay()) { - let nil_str = Ptr::new(Tag::Expr(Str), self.zero()); + let nil_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); Ok((self.intern_nil(), nil_str)) } else { let Some(idx) = ptr.pay().get_hash4() else { @@ -754,7 +854,7 @@ impl Store { /// Warning: without cache hits, this function might blow up Rust's recursion /// depth limit. This limitation is circumvented by calling `hydrate_z_cache` /// beforehand or by using `hash_ptr` instead, which is slightly slower. - fn hash_ptr_unsafe(&self, ptr: &RawPtr) -> FWrap { + fn hash_raw_ptr_unsafe(&self, ptr: &RawPtr) -> FWrap { match ptr { RawPtr::Atom(idx) => FWrap(*self.expect_f(*idx)), RawPtr::Hash3(idx) => { @@ -764,7 +864,7 @@ impl Store { let children_ptrs = self.expect_raw_ptrs::<3>(*idx); let mut children_zs = [F::ZERO; 3]; for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; + children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; } let z = FWrap(self.poseidon_cache.hash3(&children_zs)); self.z_cache.insert(*ptr, Box::new(z)); @@ -779,7 +879,7 @@ impl Store { let children_ptrs = self.expect_raw_ptrs::<4>(*idx); let mut children_zs = [F::ZERO; 4]; for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; + children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; } let z = FWrap(self.poseidon_cache.hash4(&children_zs)); self.z_cache.insert(*ptr, Box::new(z)); @@ -794,7 +894,7 @@ impl Store { let children_ptrs = self.expect_raw_ptrs::<6>(*idx); let mut children_zs = [F::ZERO; 6]; for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; + children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; } let z = FWrap(self.poseidon_cache.hash6(&children_zs)); self.z_cache.insert(*ptr, Box::new(z)); @@ -809,7 +909,7 @@ impl Store { let children_ptrs = self.expect_raw_ptrs::<8>(*idx); let mut children_zs = [F::ZERO; 8]; for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_ptr_unsafe(child_ptr).0; + children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; } let z = FWrap(self.poseidon_cache.hash8(&children_zs)); self.z_cache.insert(*ptr, Box::new(z)); @@ -829,7 +929,7 @@ impl Store { fn hydrate_z_cache_with_ptrs(&self, ptrs: &[&RawPtr]) { ptrs.chunks(256).for_each(|chunk| { chunk.par_iter().for_each(|ptr| { - self.hash_ptr_unsafe(ptr); + self.hash_raw_ptr_unsafe(ptr); }); }); } @@ -857,11 +957,11 @@ impl Store { /// Safe version of `hash_ptr_unsafe` that doesn't hit a stack overflow by /// precomputing the pointers that need to be hashed in order to hash the /// provided `ptr` - pub fn hash_ptr(&self, ptr: &RawPtr) -> FWrap { + pub fn hash_raw_ptr(&self, ptr: &RawPtr) -> FWrap { if self.is_below_safe_threshold() { // just run `hash_ptr_unsafe` for extra speed when the dehydrated // queue is small enough - return self.hash_ptr_unsafe(ptr); + return self.hash_raw_ptr_unsafe(ptr); } let mut ptrs: IndexSet<&RawPtr> = IndexSet::default(); let mut stack = vec![ptr]; @@ -908,18 +1008,38 @@ impl Store { ptrs.reverse(); self.hydrate_z_cache_with_ptrs(&ptrs.into_iter().collect::>()); // Now it's okay to call `hash_ptr_unsafe` - self.hash_ptr_unsafe(ptr) + self.hash_raw_ptr_unsafe(ptr) + } + + pub fn hash_ptr(&self, ptr: &Ptr) -> ZPtr { + ZPtr::from_parts(*ptr.tag(), self.hash_raw_ptr(ptr.pay()).0) } /// Constructs a vector of scalars that correspond to tags and hashes computed /// from a slice of `Ptr`s turned into `ZPtr`s - pub fn to_scalar_vector(&self, ptrs: &[RawPtr]) -> Vec { - ptrs.iter().map(|ptr| self.hash_ptr(ptr).0).collect() + pub fn to_scalar_vector(&self, ptrs: &[Ptr]) -> Vec { + ptrs.iter() + .fold(Vec::with_capacity(2 * ptrs.len()), |mut acc, ptr| { + let tag = ptr.tag().to_field(); + let pay = self.hash_raw_ptr(ptr.pay()).0; + acc.push(tag); + acc.push(pay); + acc + }) + } + + pub fn to_scalar_vector_raw(&self, ptrs: &[RawPtr]) -> Vec { + ptrs.iter().map(|ptr| self.hash_raw_ptr(ptr).0).collect() } /// Equality of the content-addressed versions of two pointers #[inline] - pub fn ptr_eq(&self, a: &RawPtr, b: &RawPtr) -> bool { + pub fn raw_ptr_eq(&self, a: &RawPtr, b: &RawPtr) -> bool { + self.hash_raw_ptr(a) == self.hash_raw_ptr(b) + } + + #[inline] + pub fn ptr_eq(&self, a: &Ptr, b: &Ptr) -> bool { self.hash_ptr(a) == self.hash_ptr(b) } @@ -927,15 +1047,19 @@ impl Store { /// `inverse_z_cache`. If the mapping is not there, returns an atom pointer /// with the same tag and value #[inline] - pub fn to_ptr(&self, z: &FWrap) -> RawPtr { + pub fn to_raw_ptr(&self, z: &FWrap) -> RawPtr { self.inverse_z_cache .get(z) .cloned() .unwrap_or_else(|| self.opaque(*z)) } + + #[inline] + pub fn to_ptr(&self, z_ptr: &ZPtr) -> Ptr { + Ptr::new(*z_ptr.tag(), self.to_raw_ptr(&FWrap(*z_ptr.value()))) + } } -/* impl Ptr { pub fn fmt_to_string(&self, store: &Store, state: &State) -> String { match self.tag() { @@ -970,6 +1094,7 @@ impl Ptr { } Char => { if let Some(c) = self + .pay() .get_atom() .map(|idx| store.expect_f(idx)) .and_then(F::to_char) @@ -999,7 +1124,7 @@ impl Ptr { } } Num => { - if let Some(f) = self.get_atom().map(|idx| store.expect_f(idx)) { + if let Some(f) = self.pay().get_atom().map(|idx| store.expect_f(idx)) { if let Some(u) = f.to_u64() { u.to_string() } else { @@ -1011,6 +1136,7 @@ impl Ptr { } U64 => { if let Some(u) = self + .pay() .get_atom() .map(|idx| store.expect_f(idx)) .and_then(F::to_u64) @@ -1020,10 +1146,10 @@ impl Ptr { "".into() } } - Fun => match self.get_index4() { + Fun => match self.pay().get_hash8() { None => "".into(), Some(idx) => { - if let Some((vars, body, ..)) = store.fetch_4_ptrs(idx) { + if let Some([vars, body, _, _]) = store.fetch_ptrs::<8, 4>(idx) { match vars.tag() { Tag::Expr(Nil) => { format!("", body.fmt_to_string(store, state)) @@ -1042,10 +1168,10 @@ impl Ptr { } } }, - Thunk => match self.get_index2() { + Thunk => match self.pay().get_hash4() { None => "".into(), Some(idx) => { - if let Some((val, cont)) = store.fetch_2_ptrs(idx) { + if let Some([val, cont]) = store.fetch_ptrs::<4, 2>(idx) { format!( "Thunk{{ value: {} => cont: {} }}", val.fmt_to_string(store, state), @@ -1056,7 +1182,7 @@ impl Ptr { } } }, - Comm => match self.get_atom() { + Comm => match self.pay().get_atom() { Some(idx) => { let f = store.expect_f(idx); if store.comms.get(&FWrap(*f)).is_some() { @@ -1067,10 +1193,10 @@ impl Ptr { } None => "".into(), }, - Cproc => match self.get_index2() { + Cproc => match self.pay().get_hash4() { None => "".into(), Some(idx) => { - if let Some((cproc_name, args)) = store.fetch_2_ptrs(idx) { + if let Some([cproc_name, args]) = store.fetch_ptrs::<4, 2>(idx) { format!( "", cproc_name.fmt_to_string(store, state), @@ -1129,10 +1255,10 @@ impl Ptr { store: &Store, state: &State, ) -> String { - match self.get_index4() { + match self.pay().get_hash8() { None => format!(""), Some(idx) => { - if let Some((a, cont, ..)) = store.fetch_4_ptrs(idx) { + if let Some([a, cont, _, _]) = store.fetch_ptrs::<8, 4>(idx) { format!( "{name}{{ {field}: {}, continuation: {} }}", a.fmt_to_string(store, state), @@ -1152,10 +1278,10 @@ impl Ptr { store: &Store, state: &State, ) -> String { - match self.get_index4() { + match self.pay().get_hash8() { None => format!(""), Some(idx) => { - if let Some((a, b, cont, _)) = store.fetch_4_ptrs(idx) { + if let Some([a, b, cont, _]) = store.fetch_ptrs::<8, 4>(idx) { let (fa, fb) = fields; format!( "{name}{{ {fa}: {}, {fb}: {}, continuation: {} }}", @@ -1177,10 +1303,10 @@ impl Ptr { store: &Store, state: &State, ) -> String { - match self.get_index4() { + match self.pay().get_hash8() { None => format!(""), Some(idx) => { - if let Some((a, b, c, cont)) = store.fetch_4_ptrs(idx) { + if let Some([a, b, c, cont]) = store.fetch_ptrs::<8, 4>(idx) { let (fa, fb, fc) = fields; format!( "{name}{{ {fa}: {}, {fb}: {}, {fc}: {}, continuation: {} }}", @@ -1196,4 +1322,3 @@ impl Ptr { } } } -*/ From 963c83607b6f177e95f47d22597f126aaa029ed3 Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Sat, 30 Dec 2023 10:34:10 -0300 Subject: [PATCH 04/12] Integrated raw_store into store --- src/cli/repl/meta_cmd.rs | 19 +- src/cli/repl/mod.rs | 4 +- src/cli/zstore.rs | 40 +- src/coprocessor/gadgets.rs | 6 +- src/coprocessor/mod.rs | 6 +- src/lem/eval.rs | 8 +- src/lem/interpreter.rs | 71 +- src/lem/mod.rs | 6 +- src/lem/pointers.rs | 128 +++- src/lem/raw_pointers.rs | 189 ----- src/lem/raw_store.rs | 1324 ----------------------------------- src/lem/store.rs | 990 ++++++++++++-------------- src/lem/tests/nivc_steps.rs | 4 +- 13 files changed, 612 insertions(+), 2183 deletions(-) delete mode 100644 src/lem/raw_pointers.rs delete mode 100644 src/lem/raw_store.rs diff --git a/src/cli/repl/meta_cmd.rs b/src/cli/repl/meta_cmd.rs index 3125c367b0..5a016d59be 100644 --- a/src/cli/repl/meta_cmd.rs +++ b/src/cli/repl/meta_cmd.rs @@ -18,7 +18,7 @@ use crate::{ lem::{ eval::evaluate_with_env_and_cont, multiframe::MultiFrame, - pointers::{Ptr, ZPtr}, + pointers::{Ptr, RawPtr, ZPtr}, Tag, }, package::{Package, SymbolRef}, @@ -275,13 +275,13 @@ impl MetaCmd { let (second_io, ..) = repl .eval_expr(second) .with_context(|| "evaluating second arg")?; - let Ptr::Atom(Tag::Expr(ExprTag::Num), secret) = first_io[0] else { + let (Tag::Expr(ExprTag::Num), RawPtr::Atom(secret)) = first_io[0].parts() else { bail!( "Secret must be a number. Got {}", first_io[0].fmt_to_string(&repl.store, &repl.state.borrow()) ) }; - let secret = *repl.store.expect_f(secret); + let secret = *repl.store.expect_f(*secret); repl.hide(secret, second_io[0])?; Ok(()) }, @@ -613,10 +613,10 @@ impl MetaCmd { .get_result() .expect("evaluation result must have been set"); let (_, comm) = repl.store.car_cdr(result)?; - let Ptr::Atom(Tag::Expr(ExprTag::Comm), hash) = comm else { + let (Tag::Expr(ExprTag::Comm), RawPtr::Atom(hash)) = comm.parts() else { bail!("Second component of a chain must be a commitment") }; - let hash = *repl.store.expect_f(hash); + let hash = *repl.store.expect_f(*hash); // retrieve from store to persist let (secret, fun) = repl .store @@ -892,10 +892,11 @@ impl MetaCmd { let (fun, rest) = repl.store.car_cdr(ptcl)?; - let (Ptr::Atom(Tag::Expr(ExprTag::Num), rc_idx), _) = repl.store.car_cdr(&rest)? else { + let (car, _) = repl.store.car_cdr(&rest)?; + let (Tag::Expr(ExprTag::Num), RawPtr::Atom(rc_idx)) = car.parts() else { bail!("Reduction count must be a Num") }; - let Some(rc) = repl.store.expect_f(rc_idx).to_u64().map(|u| u as usize) else { + let Some(rc) = repl.store.expect_f(*rc_idx).to_u64().map(|u| u as usize) else { bail!("Invalid value for reduction count") }; Ok((fun, rc)) @@ -938,13 +939,13 @@ impl MetaCmd { .eval_expr_with_env(apply_call, repl.store.intern_nil()) .with_context(|| "evaluating protocol function call")?; - let Ptr::Tuple2(Tag::Expr(ExprTag::Cons), idx) = &io[0] else { + let (Tag::Expr(ExprTag::Cons), RawPtr::Hash4(idx)) = &io[0].parts() else { bail!( "Protocol function must return a pair. Got {}", io[0].fmt_to_string(&repl.store, &repl.state.borrow()) ) }; - let (pre_verify, post_verify) = repl.store.fetch_2_ptrs(*idx).unwrap(); + let [pre_verify, post_verify] = &repl.store.expect_2_ptrs(*idx); if pre_verify.is_nil() { bail!("Pre-verification predicate rejected the input") diff --git a/src/cli/repl/mod.rs b/src/cli/repl/mod.rs index 8facbf69a0..724d9d22a4 100644 --- a/src/cli/repl/mod.rs +++ b/src/cli/repl/mod.rs @@ -26,7 +26,7 @@ use crate::{ eval::{evaluate_simple_with_env, evaluate_with_env}, interpreter::Frame, multiframe::MultiFrame, - pointers::Ptr, + pointers::{Ptr, RawPtr}, store::Store, Tag, }, @@ -448,7 +448,7 @@ impl Repl { let (expr_io, ..) = self .eval_expr(expr) .with_context(|| "evaluating first arg")?; - let Ptr::Atom(Tag::Expr(ExprTag::Num), hash_idx) = &expr_io[0] else { + let (Tag::Expr(ExprTag::Num), RawPtr::Atom(hash_idx)) = &expr_io[0].parts() else { bail!("hash must be a number") }; Ok(self.store.expect_f(*hash_idx)) diff --git a/src/cli/zstore.rs b/src/cli/zstore.rs index c295a837d7..65d148639c 100644 --- a/src/cli/zstore.rs +++ b/src/cli/zstore.rs @@ -5,7 +5,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use crate::{ field::{FWrap, LurkField}, lem::{ - pointers::{Ptr, ZPtr}, + pointers::{Ptr, RawPtr, ZPtr}, store::Store, }, }; @@ -38,17 +38,21 @@ impl ZDag { if let Some(z_ptr) = cache.get(ptr) { *z_ptr } else { - let z_ptr = match ptr { - Ptr::Atom(tag, idx) => { + let tag = ptr.tag(); + let z_ptr = match ptr.pay() { + RawPtr::Atom(idx) => { let f = store.expect_f(*idx); let z_ptr = ZPtr::from_parts(*tag, *f); self.0.insert(z_ptr, ZPtrType::Atom); z_ptr } - Ptr::Tuple2(tag, idx) => { - let (a, b) = store.expect_2_ptrs(*idx); - let a = self.populate_with(a, store, cache); - let b = self.populate_with(b, store, cache); + RawPtr::Hash3(_) => { + todo!() + } + RawPtr::Hash4(idx) => { + let [a, b] = store.expect_2_ptrs(*idx); + let a = self.populate_with(&a, store, cache); + let b = self.populate_with(&b, store, cache); let z_ptr = ZPtr::from_parts( *tag, store.poseidon_cache.hash4(&[ @@ -61,11 +65,11 @@ impl ZDag { self.0.insert(z_ptr, ZPtrType::Tuple2(a, b)); z_ptr } - Ptr::Tuple3(tag, idx) => { - let (a, b, c) = store.expect_3_ptrs(*idx); - let a = self.populate_with(a, store, cache); - let b = self.populate_with(b, store, cache); - let c = self.populate_with(c, store, cache); + RawPtr::Hash6(idx) => { + let [a, b, c] = store.expect_3_ptrs(*idx); + let a = self.populate_with(&a, store, cache); + let b = self.populate_with(&b, store, cache); + let c = self.populate_with(&c, store, cache); let z_ptr = ZPtr::from_parts( *tag, store.poseidon_cache.hash6(&[ @@ -80,12 +84,12 @@ impl ZDag { self.0.insert(z_ptr, ZPtrType::Tuple3(a, b, c)); z_ptr } - Ptr::Tuple4(tag, idx) => { - let (a, b, c, d) = store.expect_4_ptrs(*idx); - let a = self.populate_with(a, store, cache); - let b = self.populate_with(b, store, cache); - let c = self.populate_with(c, store, cache); - let d = self.populate_with(d, store, cache); + RawPtr::Hash8(idx) => { + let [a, b, c, d] = store.expect_4_ptrs(*idx); + let a = self.populate_with(&a, store, cache); + let b = self.populate_with(&b, store, cache); + let c = self.populate_with(&c, store, cache); + let d = self.populate_with(&d, store, cache); let z_ptr = ZPtr::from_parts( *tag, store.poseidon_cache.hash8(&[ diff --git a/src/coprocessor/gadgets.rs b/src/coprocessor/gadgets.rs index 473c5fa38a..371a078b03 100644 --- a/src/coprocessor/gadgets.rs +++ b/src/coprocessor/gadgets.rs @@ -178,7 +178,7 @@ pub(crate) fn deconstruct_tuple2>( ) -> Result<(AllocatedPtr, AllocatedPtr), SynthesisError> { let (a, b) = if not_dummy.get_value() == Some(true) { let idx = get_ptr(tuple, store)?.get_index2().expect("invalid Ptr"); - let (a, b) = store.expect_2_ptrs(idx); + let [a, b] = &store.expect_2_ptrs(idx); (store.hash_ptr(a), store.hash_ptr(b)) } else { (ZPtr::dummy(), ZPtr::dummy()) @@ -221,7 +221,7 @@ pub(crate) fn deconstruct_tuple3>( ) -> Result<(AllocatedPtr, AllocatedPtr, AllocatedPtr), SynthesisError> { let (a, b, c) = if not_dummy.get_value() == Some(true) { let idx = get_ptr(tuple, store)?.get_index3().expect("invalid Ptr"); - let (a, b, c) = store.expect_3_ptrs(idx); + let [a, b, c] = &store.expect_3_ptrs(idx); (store.hash_ptr(a), store.hash_ptr(b), store.hash_ptr(c)) } else { (ZPtr::dummy(), ZPtr::dummy(), ZPtr::dummy()) @@ -275,7 +275,7 @@ pub(crate) fn deconstruct_tuple4>( > { let (a, b, c, d) = if not_dummy.get_value() == Some(true) { let idx = get_ptr(tuple, store)?.get_index4().expect("invalid Ptr"); - let (a, b, c, d) = store.expect_4_ptrs(idx); + let [a, b, c, d] = &store.expect_4_ptrs(idx); ( store.hash_ptr(a), store.hash_ptr(b), diff --git a/src/coprocessor/mod.rs b/src/coprocessor/mod.rs index 8c711370db..beb97360d8 100644 --- a/src/coprocessor/mod.rs +++ b/src/coprocessor/mod.rs @@ -113,7 +113,7 @@ pub(crate) mod test { use super::*; use crate::circuit::gadgets::constraints::{alloc_equal, mul}; - use crate::lem::Tag as LEMTag; + use crate::lem::{pointers::RawPtr, Tag as LEMTag}; use crate::tag::{ExprTag, Tag}; use std::marker::PhantomData; @@ -209,11 +209,11 @@ pub(crate) mod test { } fn evaluate(&self, s: &Store, args: &[Ptr], env: &Ptr, cont: &Ptr) -> Vec { - let Ptr::Atom(LEMTag::Expr(ExprTag::Num), a) = &args[0] else { + let (LEMTag::Expr(ExprTag::Num), RawPtr::Atom(a)) = args[0].parts() else { return vec![args[0], *env, s.cont_error()]; }; let a = s.expect_f(*a); - let Ptr::Atom(LEMTag::Expr(ExprTag::Num), b) = &args[1] else { + let (LEMTag::Expr(ExprTag::Num), RawPtr::Atom(b)) = args[1].parts() else { return vec![args[1], *env, s.cont_error()]; }; let b = s.expect_f(*b); diff --git a/src/lem/eval.rs b/src/lem/eval.rs index ce64d6cd89..9ad14daca6 100644 --- a/src/lem/eval.rs +++ b/src/lem/eval.rs @@ -21,7 +21,7 @@ use crate::{ use super::{ interpreter::{Frame, Hints}, - pointers::Ptr, + pointers::{Ptr, RawPtr}, store::Store, Ctrl, Func, Op, Tag, Var, }; @@ -40,9 +40,9 @@ fn get_pc>( store: &Store, lang: &Lang, ) -> usize { - match expr { - Ptr::Tuple2(Tag::Expr(Cproc), idx) => { - let (cproc, _) = store + match expr.parts() { + (Tag::Expr(Cproc), RawPtr::Hash4(idx)) => { + let [cproc, _] = &store .fetch_2_ptrs(*idx) .expect("Coprocessor expression is not interned"); let cproc_sym = store diff --git a/src/lem/interpreter.rs b/src/lem/interpreter.rs index 23d2e86317..6fb04aefac 100644 --- a/src/lem/interpreter.rs +++ b/src/lem/interpreter.rs @@ -2,7 +2,7 @@ use anyhow::{anyhow, bail, Result}; use super::{ path::Path, - pointers::Ptr, + pointers::{Ptr, RawPtr}, slot::{SlotData, Val}, store::Store, var_map::VarMap, @@ -181,16 +181,16 @@ impl Block { bindings.insert_ptr(tgt.clone(), store.zero(*tag)); } Op::Hash3Zeros(tgt, tag) => { - bindings.insert_ptr(tgt.clone(), Ptr::Atom(*tag, store.hash3zeros_idx)); + bindings.insert_ptr(tgt.clone(), Ptr::atom(*tag, store.hash3zeros_idx)); } Op::Hash4Zeros(tgt, tag) => { - bindings.insert_ptr(tgt.clone(), Ptr::Atom(*tag, store.hash4zeros_idx)); + bindings.insert_ptr(tgt.clone(), Ptr::atom(*tag, store.hash4zeros_idx)); } Op::Hash6Zeros(tgt, tag) => { - bindings.insert_ptr(tgt.clone(), Ptr::Atom(*tag, store.hash6zeros_idx)); + bindings.insert_ptr(tgt.clone(), Ptr::atom(*tag, store.hash6zeros_idx)); } Op::Hash8Zeros(tgt, tag) => { - bindings.insert_ptr(tgt.clone(), Ptr::Atom(*tag, store.hash8zeros_idx)); + bindings.insert_ptr(tgt.clone(), Ptr::atom(*tag, store.hash8zeros_idx)); } Op::Lit(tgt, lit) => { bindings.insert_ptr(tgt.clone(), lit.to_ptr(store)); @@ -229,9 +229,9 @@ impl Block { bindings.insert_bool(tgt.clone(), a || b); } Op::Add(tgt, a, b) => { - let a = bindings.get_ptr(a)?; - let b = bindings.get_ptr(b)?; - let c = if let (Ptr::Atom(_, f), Ptr::Atom(_, g)) = (a, b) { + let a = *bindings.get_ptr(a)?.pay(); + let b = *bindings.get_ptr(b)?.pay(); + let c = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let (f, g) = (store.expect_f(f), store.expect_f(g)); store.intern_atom(Tag::Expr(Num), *f + *g) } else { @@ -240,9 +240,9 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::Sub(tgt, a, b) => { - let a = bindings.get_ptr(a)?; - let b = bindings.get_ptr(b)?; - let c = if let (Ptr::Atom(_, f), Ptr::Atom(_, g)) = (a, b) { + let a = *bindings.get_ptr(a)?.pay(); + let b = *bindings.get_ptr(b)?.pay(); + let c = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let (f, g) = (store.expect_f(f), store.expect_f(g)); store.intern_atom(Tag::Expr(Num), *f - *g) } else { @@ -251,9 +251,9 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::Mul(tgt, a, b) => { - let a = bindings.get_ptr(a)?; - let b = bindings.get_ptr(b)?; - let c = if let (Ptr::Atom(_, f), Ptr::Atom(_, g)) = (a, b) { + let a = *bindings.get_ptr(a)?.pay(); + let b = *bindings.get_ptr(b)?.pay(); + let c = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let (f, g) = (store.expect_f(f), store.expect_f(g)); store.intern_atom(Tag::Expr(Num), *f * *g) } else { @@ -262,9 +262,9 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::Div(tgt, a, b) => { - let a = bindings.get_ptr(a)?; - let b = bindings.get_ptr(b)?; - let c = if let (Ptr::Atom(_, f), Ptr::Atom(_, g)) = (a, b) { + let a = *bindings.get_ptr(a)?.pay(); + let b = *bindings.get_ptr(b)?.pay(); + let c = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let (f, g) = (store.expect_f(f), store.expect_f(g)); if g == &F::ZERO { bail!("Can't divide by zero") @@ -276,9 +276,9 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::Lt(tgt, a, b) => { - let a = bindings.get_ptr(a)?; - let b = bindings.get_ptr(b)?; - let c = if let (Ptr::Atom(_, f_idx), Ptr::Atom(_, g_idx)) = (a, b) { + let a = *bindings.get_ptr(a)?.pay(); + let b = *bindings.get_ptr(b)?.pay(); + let c = if let (RawPtr::Atom(f_idx), RawPtr::Atom(g_idx)) = (a, b) { let f = *store.expect_f(f_idx); let g = *store.expect_f(g_idx); let diff = f - g; @@ -301,8 +301,8 @@ impl Block { } Op::Trunc(tgt, a, n) => { assert!(*n <= 64); - let a = bindings.get_ptr(a)?; - let c = if let Ptr::Atom(_, f_idx) = a { + let a = *bindings.get_ptr(a)?.pay(); + let c = if let RawPtr::Atom(f_idx) = a { let f = *store.expect_f(f_idx); hints.bit_decomp.push(Some(SlotData { vals: vec![Val::Num(f_idx)], @@ -315,9 +315,9 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::DivRem64(tgt, a, b) => { - let a = bindings.get_ptr(a)?; - let b = bindings.get_ptr(b)?; - let (c1, c2) = if let (Ptr::Atom(_, f), Ptr::Atom(_, g)) = (a, b) { + let a = *bindings.get_ptr(a)?.pay(); + let b = *bindings.get_ptr(b)?.pay(); + let (c1, c2) = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let f = *store.expect_f(f); let g = *store.expect_f(g); if g == F::ZERO { @@ -372,10 +372,9 @@ impl Block { let Some(idx) = img_ptr.get_index2() else { bail!("{img} isn't a Tree2 pointer"); }; - let Some((a, b)) = store.fetch_2_ptrs(idx) else { + let Some(preimg_ptrs) = store.fetch_2_ptrs(idx) else { bail!("Couldn't fetch {img}'s children") }; - let preimg_ptrs = [*a, *b]; for (var, ptr) in preimg.iter().zip(preimg_ptrs.iter()) { bindings.insert_ptr(var.clone(), *ptr); } @@ -387,10 +386,9 @@ impl Block { let Some(idx) = img_ptr.get_index3() else { bail!("{img} isn't a Tree3 pointer"); }; - let Some((a, b, c)) = store.fetch_3_ptrs(idx) else { + let Some(preimg_ptrs) = store.fetch_3_ptrs(idx) else { bail!("Couldn't fetch {img}'s children") }; - let preimg_ptrs = [*a, *b, *c]; for (var, ptr) in preimg.iter().zip(preimg_ptrs.iter()) { bindings.insert_ptr(var.clone(), *ptr); } @@ -402,10 +400,9 @@ impl Block { let Some(idx) = img_ptr.get_index4() else { bail!("{img} isn't a Tree4 pointer"); }; - let Some((a, b, c, d)) = store.fetch_4_ptrs(idx) else { + let Some(preimg_ptrs) = store.fetch_4_ptrs(idx) else { bail!("Couldn't fetch {img}'s children") }; - let preimg_ptrs = [*a, *b, *c, *d]; for (var, ptr) in preimg.iter().zip(preimg_ptrs.iter()) { bindings.insert_ptr(var.clone(), *ptr); } @@ -414,20 +411,22 @@ impl Block { } Op::Hide(tgt, sec, src) => { let src_ptr = bindings.get_ptr(src)?; - let Ptr::Atom(Tag::Expr(Num), secret_idx) = bindings.get_ptr(sec)? else { + let sec_ptr = bindings.get_ptr(sec)?; + let (Tag::Expr(Num), RawPtr::Atom(secret_idx)) = sec_ptr.parts() else { bail!("{sec} is not a numeric pointer") }; - let secret = *store.expect_f(secret_idx); + let secret = *store.expect_f(*secret_idx); let tgt_ptr = store.hide(secret, src_ptr); - let vals = vec![Val::Num(secret_idx), Val::Pointer(src_ptr)]; + let vals = vec![Val::Num(*secret_idx), Val::Pointer(src_ptr)]; hints.commitment.push(Some(SlotData { vals })); bindings.insert_ptr(tgt.clone(), tgt_ptr); } Op::Open(tgt_secret, tgt_ptr, comm) => { - let Ptr::Atom(Tag::Expr(Comm), hash) = bindings.get_ptr(comm)? else { + let comm_ptr = bindings.get_ptr(comm)?; + let (Tag::Expr(Comm), RawPtr::Atom(hash)) = comm_ptr.parts() else { bail!("{comm} is not a comm pointer") }; - let hash = *store.expect_f(hash); + let hash = *store.expect_f(*hash); let Some((secret, ptr)) = store.open(hash).cloned() else { bail!("No committed data for hash {}", &hash.hex_digits()) }; diff --git a/src/lem/mod.rs b/src/lem/mod.rs index 836ef4c4ba..042386ad9b 100644 --- a/src/lem/mod.rs +++ b/src/lem/mod.rs @@ -66,8 +66,6 @@ mod macros; pub mod multiframe; mod path; pub mod pointers; -pub mod raw_pointers; -pub mod raw_store; mod slot; pub mod store; mod var_map; @@ -122,6 +120,10 @@ impl TryFrom for Tag { Ok(Tag::Expr(tag)) } else if let Ok(tag) = ContTag::try_from(val) { Ok(Tag::Cont(tag)) + } else if let Ok(tag) = Op1::try_from(val) { + Ok(Tag::Op1(tag)) + } else if let Ok(tag) = Op2::try_from(val) { + Ok(Tag::Op2(tag)) } else { bail!("Invalid u16 for Tag: {val}") } diff --git a/src/lem/pointers.rs b/src/lem/pointers.rs index 241700bc3b..bedd755506 100644 --- a/src/lem/pointers.rs +++ b/src/lem/pointers.rs @@ -7,6 +7,65 @@ use crate::{ use super::Tag; +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub enum RawPtr { + Atom(usize), + Hash3(usize), + Hash4(usize), + Hash6(usize), + Hash8(usize), +} + +impl RawPtr { + #[inline] + pub fn is_hash(&self) -> bool { + matches!( + self, + RawPtr::Hash3(..) | RawPtr::Hash4(..) | RawPtr::Hash6(..) | RawPtr::Hash8(..) + ) + } + + #[inline] + pub fn get_atom(&self) -> Option { + match self { + RawPtr::Atom(x) => Some(*x), + _ => None, + } + } + + #[inline] + pub fn get_hash3(&self) -> Option { + match self { + RawPtr::Hash3(x) => Some(*x), + _ => None, + } + } + + #[inline] + pub fn get_hash4(&self) -> Option { + match self { + RawPtr::Hash4(x) => Some(*x), + _ => None, + } + } + + #[inline] + pub fn get_hash6(&self) -> Option { + match self { + RawPtr::Hash6(x) => Some(*x), + _ => None, + } + } + + #[inline] + pub fn get_hash8(&self) -> Option { + match self { + RawPtr::Hash8(x) => Some(*x), + _ => None, + } + } +} + /// `Ptr` is the main piece of data LEMs operate on. We can think of a pointer /// as a building block for trees that represent Lurk data. A pointer can be a /// atom that contains data encoded as an element of a `LurkField` or it can have @@ -19,20 +78,31 @@ use super::Tag; /// hashing operations can plug any tag to the resulting pointer. Thus, the /// number of children have to be made explicit as the `Ptr` enum. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] -pub enum Ptr { - Atom(Tag, usize), - Tuple2(Tag, usize), - Tuple3(Tag, usize), - Tuple4(Tag, usize), +pub struct Ptr { + tag: Tag, + pay: RawPtr, } impl Ptr { + #[inline] + pub fn new(tag: Tag, pay: RawPtr) -> Self { + Ptr { tag, pay } + } + + #[inline] pub fn tag(&self) -> &Tag { - match self { - Ptr::Atom(tag, _) | Ptr::Tuple2(tag, _) | Ptr::Tuple3(tag, _) | Ptr::Tuple4(tag, _) => { - tag - } - } + &self.tag + } + + #[inline] + pub fn pay(&self) -> &RawPtr { + &self.pay + } + + #[inline] + pub fn parts(&self) -> (&Tag, &RawPtr) { + let Ptr { tag, pay } = self; + (tag, pay) } #[inline] @@ -77,48 +147,34 @@ impl Ptr { #[inline] pub fn cast(self, tag: Tag) -> Self { - match self { - Ptr::Atom(_, x) => Ptr::Atom(tag, x), - Ptr::Tuple2(_, x) => Ptr::Tuple2(tag, x), - Ptr::Tuple3(_, x) => Ptr::Tuple3(tag, x), - Ptr::Tuple4(_, x) => Ptr::Tuple4(tag, x), - } - } - - #[inline] - pub fn is_tuple(&self) -> bool { - matches!(self, Ptr::Tuple2(..) | Ptr::Tuple3(..) | Ptr::Tuple4(..)) + Ptr { tag, pay: self.pay } } #[inline] pub fn get_atom(&self) -> Option { - match self { - Ptr::Atom(_, x) => Some(*x), - _ => None, - } + self.pay().get_atom() } #[inline] pub fn get_index2(&self) -> Option { - match self { - Ptr::Tuple2(_, x) => Some(*x), - _ => None, - } + self.pay().get_hash4() } #[inline] pub fn get_index3(&self) -> Option { - match self { - Ptr::Tuple3(_, x) => Some(*x), - _ => None, - } + self.pay().get_hash6() } #[inline] pub fn get_index4(&self) -> Option { - match self { - Ptr::Tuple4(_, x) => Some(*x), - _ => None, + self.pay().get_hash8() + } + + #[inline] + pub fn atom(tag: Tag, idx: usize) -> Ptr { + Ptr { + tag, + pay: RawPtr::Atom(idx), } } } diff --git a/src/lem/raw_pointers.rs b/src/lem/raw_pointers.rs deleted file mode 100644 index f89217745b..0000000000 --- a/src/lem/raw_pointers.rs +++ /dev/null @@ -1,189 +0,0 @@ -use serde::{Deserialize, Serialize}; - -use crate::tag::ExprTag::{Cons, Fun, Nil, Num, Str, Sym}; - -use super::Tag; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] -pub enum RawPtr { - Atom(usize), - Hash3(usize), - Hash4(usize), - Hash6(usize), - Hash8(usize), -} - -impl RawPtr { - #[inline] - pub fn is_hash(&self) -> bool { - matches!( - self, - RawPtr::Hash3(..) | RawPtr::Hash4(..) | RawPtr::Hash6(..) | RawPtr::Hash8(..) - ) - } - - #[inline] - pub fn get_atom(&self) -> Option { - match self { - RawPtr::Atom(x) => Some(*x), - _ => None, - } - } - - #[inline] - pub fn get_hash3(&self) -> Option { - match self { - RawPtr::Hash3(x) => Some(*x), - _ => None, - } - } - - #[inline] - pub fn get_hash4(&self) -> Option { - match self { - RawPtr::Hash4(x) => Some(*x), - _ => None, - } - } - - #[inline] - pub fn get_hash6(&self) -> Option { - match self { - RawPtr::Hash6(x) => Some(*x), - _ => None, - } - } - - #[inline] - pub fn get_hash8(&self) -> Option { - match self { - RawPtr::Hash8(x) => Some(*x), - _ => None, - } - } -} - -/// `Ptr` is the main piece of data LEMs operate on. We can think of a pointer -/// as a building block for trees that represent Lurk data. A pointer can be a -/// atom that contains data encoded as an element of a `LurkField` or it can have -/// children. For performance, the children of a pointer are stored on an -/// `IndexSet` and the resulding index is carried by the pointer itself. -/// -/// A pointer also has a tag, which says what kind of data it encodes. On -/// previous implementations, the tag would be used to infer the number of -/// children a pointer has. However, LEMs require extra flexibility because LEM -/// hashing operations can plug any tag to the resulting pointer. Thus, the -/// number of children have to be made explicit as the `Ptr` enum. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] -pub struct Ptr { - tag: Tag, - pay: RawPtr, -} - -impl Ptr { - pub fn new(tag: Tag, pay: RawPtr) -> Self { - Ptr { tag, pay } - } - - pub fn tag(&self) -> &Tag { - &self.tag - } - - pub fn pay(&self) -> &RawPtr { - &self.pay - } - - #[inline] - pub fn has_tag(&self, tag: &Tag) -> bool { - self.tag() == tag - } - - #[inline] - pub fn has_tag_in(&self, tags: &[Tag]) -> bool { - tags.contains(self.tag()) - } - - #[inline] - pub fn is_sym(&self) -> bool { - self.has_tag(&Tag::Expr(Sym)) - } - - #[inline] - pub fn is_num(&self) -> bool { - self.has_tag(&Tag::Expr(Num)) - } - - #[inline] - pub fn is_str(&self) -> bool { - self.has_tag(&Tag::Expr(Str)) - } - - #[inline] - pub fn is_fun(&self) -> bool { - self.has_tag(&Tag::Expr(Fun)) - } - - #[inline] - pub fn is_nil(&self) -> bool { - self.has_tag(&Tag::Expr(Nil)) - } - - #[inline] - pub fn is_list(&self) -> bool { - self.has_tag_in(&[Tag::Expr(Cons), Tag::Expr(Nil)]) - } - - #[inline] - pub fn cast(self, tag: Tag) -> Self { - Ptr { tag, pay: self.pay } - } - - #[inline] - pub fn get_atom(&self) -> Option { - self.pay().get_atom() - } - - #[inline] - pub fn get_index2(&self) -> Option { - self.pay().get_hash4() - } - - #[inline] - pub fn get_index3(&self) -> Option { - self.pay().get_hash6() - } - - #[inline] - pub fn get_index4(&self) -> Option { - self.pay().get_hash8() - } - - #[inline] - pub fn atom(tag: Tag, idx: usize) -> Ptr { - Ptr { - tag, - pay: RawPtr::Atom(idx), - } - } -} - -/// A `ZPtr` is the result of "hydrating" a `Ptr`. This process is better -/// explained in the store but, in short, we want to know the Poseidon hash of -/// the children of a `Ptr`. -/// -/// `ZPtr`s are used mainly for proofs, but they're also useful when we want -/// to content-address a store. -/// -/// An important note is that computing the respective `ZPtr` of a `Ptr` can be -/// expensive because of the Poseidon hashes. That's why we operate on `Ptr`s -/// when interpreting LEMs and delay the need for `ZPtr`s as much as possible. -pub type ZPtr = crate::z_data::z_ptr::ZPtr; - -/* -impl ZPtr { - #[inline] - pub fn dummy() -> Self { - Self(Tag::Expr(Nil), F::ZERO) - } -} -*/ diff --git a/src/lem/raw_store.rs b/src/lem/raw_store.rs deleted file mode 100644 index 6c9d906d05..0000000000 --- a/src/lem/raw_store.rs +++ /dev/null @@ -1,1324 +0,0 @@ -use anyhow::{bail, Context, Result}; -use arc_swap::ArcSwap; -use bellpepper::util_cs::witness_cs::SizedWitness; -use elsa::{ - sync::index_set::FrozenIndexSet, - sync::{FrozenMap, FrozenVec}, -}; -use indexmap::IndexSet; -use neptune::Poseidon; -use nom::{sequence::preceded, Parser}; -use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; -use std::{cell::RefCell, rc::Rc, sync::Arc}; - -use crate::{ - field::{FWrap, LurkField}, - hash::{InversePoseidonCache, PoseidonCache}, - lem::Tag, - parser::{syntax, Error, Span}, - state::{lurk_sym, user_sym, State}, - symbol::Symbol, - syntax::Syntax, - tag::ContTag::{ - self, Binop, Binop2, Call, Call0, Call2, Dummy, Emit, If, Let, LetRec, Lookup, Outermost, - Tail, Terminal, Unop, - }, - tag::ExprTag::{Char, Comm, Cons, Cproc, Fun, Key, Nil, Num, Str, Sym, Thunk, U64}, - tag::Tag as TagTrait, -}; - -use super::raw_pointers::{Ptr, RawPtr, ZPtr}; - -#[derive(Debug)] -pub struct Store { - f_elts: FrozenIndexSet>>, - hash3: FrozenIndexSet>, - hash4: FrozenIndexSet>, - hash6: FrozenIndexSet>, - hash8: FrozenIndexSet>, - - string_ptr_cache: FrozenMap>, - symbol_ptr_cache: FrozenMap>, - - ptr_string_cache: FrozenMap, - ptr_symbol_cache: FrozenMap>, - - comms: FrozenMap, Box<(F, Ptr)>>, // hash -> (secret, src) - - pub poseidon_cache: PoseidonCache, - pub inverse_poseidon_cache: InversePoseidonCache, - - dehydrated: ArcSwap>>, - z_cache: FrozenMap>>, - inverse_z_cache: FrozenMap, Box>, - - // cached indices for the hashes of 3, 4, 6 and 8 padded zeros - pub hash3zeros_idx: usize, - pub hash4zeros_idx: usize, - pub hash6zeros_idx: usize, - pub hash8zeros_idx: usize, -} - -impl Default for Store { - fn default() -> Self { - let poseidon_cache = PoseidonCache::default(); - let hash3zeros = poseidon_cache.hash3(&[F::ZERO; 3]); - let hash4zeros = poseidon_cache.hash4(&[F::ZERO; 4]); - let hash6zeros = poseidon_cache.hash6(&[F::ZERO; 6]); - let hash8zeros = poseidon_cache.hash8(&[F::ZERO; 8]); - - let f_elts = FrozenIndexSet::default(); - let (hash3zeros_idx, _) = f_elts.insert_probe(FWrap(hash3zeros).into()); - let (hash4zeros_idx, _) = f_elts.insert_probe(FWrap(hash4zeros).into()); - let (hash6zeros_idx, _) = f_elts.insert_probe(FWrap(hash6zeros).into()); - let (hash8zeros_idx, _) = f_elts.insert_probe(FWrap(hash8zeros).into()); - - Self { - f_elts, - hash3: Default::default(), - hash4: Default::default(), - hash6: Default::default(), - hash8: Default::default(), - string_ptr_cache: Default::default(), - symbol_ptr_cache: Default::default(), - ptr_string_cache: Default::default(), - ptr_symbol_cache: Default::default(), - comms: Default::default(), - poseidon_cache, - inverse_poseidon_cache: Default::default(), - dehydrated: Default::default(), - z_cache: Default::default(), - inverse_z_cache: Default::default(), - hash3zeros_idx, - hash4zeros_idx, - hash6zeros_idx, - hash8zeros_idx, - } - } -} - -impl Store { - /// Cost of poseidon hash with arity 3, including the input - #[inline] - pub fn hash3_cost(&self) -> usize { - Poseidon::new(self.poseidon_cache.constants.c3()).num_aux() + 1 - } - - /// Cost of poseidon hash with arity 4, including the input - #[inline] - pub fn hash4_cost(&self) -> usize { - Poseidon::new(self.poseidon_cache.constants.c4()).num_aux() + 1 - } - - /// Cost of poseidon hash with arity 6, including the input - #[inline] - pub fn hash6_cost(&self) -> usize { - Poseidon::new(self.poseidon_cache.constants.c6()).num_aux() + 1 - } - - /// Cost of poseidon hash with arity 8, including the input - #[inline] - pub fn hash8_cost(&self) -> usize { - Poseidon::new(self.poseidon_cache.constants.c8()).num_aux() + 1 - } - - /// Retrieves the hash of 3 padded zeros - #[inline] - pub fn hash3zeros(&self) -> &F { - self.expect_f(self.hash3zeros_idx) - } - - /// Retrieves the hash of 4 padded zeros - #[inline] - pub fn hash4zeros(&self) -> &F { - self.expect_f(self.hash4zeros_idx) - } - - /// Retrieves the hash of 6 padded zeros - #[inline] - pub fn hash6zeros(&self) -> &F { - self.expect_f(self.hash6zeros_idx) - } - - /// Retrieves the hash of 8 padded zeros - #[inline] - pub fn hash8zeros(&self) -> &F { - self.expect_f(self.hash8zeros_idx) - } - - #[inline] - pub fn intern_f(&self, f: F) -> (usize, bool) { - self.f_elts.insert_probe(Box::new(FWrap(f))) - } - - /// Creates an atom `RawPtr` which points to a cached element of the finite - /// field `F` - pub fn intern_raw_atom(&self, f: F) -> RawPtr { - let (idx, _) = self.intern_f(f); - RawPtr::Atom(idx) - } - - pub fn intern_atom(&self, tag: Tag, f: F) -> Ptr { - Ptr::new(tag, self.intern_raw_atom(f)) - } - - /// Creates a `RawPtr` that's a parent of `N` children - pub fn intern_raw_ptrs(&self, ptrs: [RawPtr; N]) -> RawPtr { - macro_rules! intern { - ($Hash:ident, $hash:ident, $n:expr) => {{ - let ptrs = unsafe { std::mem::transmute::<&[RawPtr; N], &[RawPtr; $n]>(&ptrs) }; - let (idx, inserted) = self.$hash.insert_probe(Box::new(*ptrs)); - (RawPtr::$Hash(idx), inserted) - }}; - } - let (ptr, inserted) = match N { - 3 => intern!(Hash3, hash3, 3), - 4 => intern!(Hash4, hash4, 4), - 6 => intern!(Hash6, hash6, 6), - 8 => intern!(Hash8, hash8, 8), - _ => unimplemented!(), - }; - if inserted { - // this is for `hydrate_z_cache` - self.dehydrated.load().push(Box::new(ptr)); - } - ptr - } - - /// Similar to `intern_raw_ptrs` but doesn't add the resulting pointer to - /// `dehydrated`. This function is used when converting a `ZStore` to a - /// `Store`. - pub fn intern_raw_ptrs_hydrated( - &self, - ptrs: [RawPtr; N], - z: FWrap, - ) -> RawPtr { - macro_rules! intern { - ($Hash:ident, $hash:ident, $n:expr) => {{ - let ptrs = unsafe { std::mem::transmute::<&[RawPtr; N], &[RawPtr; $n]>(&ptrs) }; - let (idx, _) = self.$hash.insert_probe(Box::new(*ptrs)); - RawPtr::$Hash(idx) - }}; - } - let ptr = match N { - 3 => intern!(Hash3, hash3, 3), - 4 => intern!(Hash4, hash4, 4), - 6 => intern!(Hash6, hash6, 6), - 8 => intern!(Hash8, hash8, 8), - _ => unimplemented!(), - }; - self.z_cache.insert(ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(ptr)); - ptr - } - - /// Creates a `Ptr` that's a parent of `N` children - pub fn intern_ptrs(&self, tag: Tag, ptrs: [Ptr; N]) -> Ptr { - macro_rules! intern { - ($n:expr) => {{ - let mut raw_ptrs = [self.raw_zero(); $n * 2]; - for i in 0..$n { - raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); - raw_ptrs[2 * i + 1] = *ptrs[i].pay(); - } - self.intern_raw_ptrs::<{ $n * 2 }>(raw_ptrs) - }}; - } - let pay = match N { - 2 => intern!(2), - 3 => intern!(3), - 4 => intern!(4), - _ => unimplemented!(), - }; - Ptr::new(tag, pay) - } - - /// Similar to `intern_ptrs` but doesn't add the resulting pointer to - /// `dehydrated`. This function is used when converting a `ZStore` to a - /// `Store`. - pub fn intern_ptrs_hydrated( - &self, - tag: Tag, - ptrs: [Ptr; N], - z: FWrap, - ) -> Ptr { - macro_rules! intern { - ($n:expr) => {{ - let mut raw_ptrs = [self.raw_zero(); $n * 2]; - for i in 0..$n { - raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); - raw_ptrs[2 * i + 1] = *ptrs[i].pay(); - } - self.intern_raw_ptrs_hydrated::<{ $n * 2 }>(raw_ptrs, z) - }}; - } - let pay = match N { - 2 => intern!(2), - 3 => intern!(3), - 4 => intern!(4), - _ => unimplemented!(), - }; - Ptr::new(tag, pay) - } - - #[inline] - pub fn fetch_f(&self, idx: usize) -> Option<&F> { - self.f_elts.get_index(idx).map(|fw| &fw.0) - } - - #[inline] - pub fn fetch_raw_ptrs(&self, idx: usize) -> Option<&[RawPtr; N]> { - macro_rules! fetch { - ($hash:ident, $n:expr) => {{ - let ptrs = self.$hash.get_index(idx)?; - let ptrs = unsafe { std::mem::transmute::<&[RawPtr; $n], &[RawPtr; N]>(ptrs) }; - Some(ptrs) - }}; - } - match N { - 3 => fetch!(hash3, 3), - 4 => fetch!(hash4, 4), - 6 => fetch!(hash6, 6), - 8 => fetch!(hash8, 8), - _ => unimplemented!(), - } - } - - #[inline] - pub fn fetch_ptrs(&self, idx: usize) -> Option<[Ptr; P]> { - assert_eq!(P * 2, N); - let raw_ptrs = self - .fetch_raw_ptrs::(idx) - .expect("Index missing from store"); - let mut ptrs = [self.dummy(); P]; - for i in 0..P { - let tag = self.fetch_tag(&raw_ptrs[2 * i])?; - ptrs[i] = Ptr::new(tag, raw_ptrs[2 * i + 1]) - } - Some(ptrs) - } - - #[inline] - pub fn expect_f(&self, idx: usize) -> &F { - self.fetch_f(idx).expect("Index missing from f_elts") - } - - #[inline] - pub fn expect_raw_ptrs(&self, idx: usize) -> &[RawPtr; N] { - self.fetch_raw_ptrs::(idx) - .expect("Index missing from store") - } - - #[inline] - pub fn expect_ptrs(&self, idx: usize) -> [Ptr; P] { - self.fetch_ptrs::(idx) - .expect("Index missing from store") - } - - // TODO remove these functions - pub fn intern_atom_hydrated(&self, tag: Tag, f: F, _: ZPtr) -> Ptr { - Ptr::new(tag, self.intern_raw_atom(f)) - } - #[inline] - pub fn intern_2_ptrs(&self, tag: Tag, a: Ptr, b: Ptr) -> Ptr { - self.intern_ptrs::<2>(tag, [a, b]) - } - #[inline] - pub fn intern_3_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr) -> Ptr { - self.intern_ptrs::<3>(tag, [a, b, c]) - } - #[inline] - pub fn intern_4_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, d: Ptr) -> Ptr { - self.intern_ptrs::<4>(tag, [a, b, c, d]) - } - #[inline] - pub fn intern_2_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, z: ZPtr) -> Ptr { - self.intern_ptrs_hydrated::<2>(tag, [a, b], FWrap(*z.value())) - } - #[inline] - pub fn intern_3_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, z: ZPtr) -> Ptr { - self.intern_ptrs_hydrated::<3>(tag, [a, b, c], FWrap(*z.value())) - } - #[inline] - pub fn intern_4_ptrs_hydrated( - &self, - tag: Tag, - a: Ptr, - b: Ptr, - c: Ptr, - d: Ptr, - z: ZPtr, - ) -> Ptr { - self.intern_ptrs_hydrated::<4>(tag, [a, b, c, d], FWrap(*z.value())) - } - #[inline] - pub fn fetch_2_ptrs(&self, idx: usize) -> Option<[Ptr; 2]> { - self.fetch_ptrs::<4, 2>(idx) - } - #[inline] - pub fn fetch_3_ptrs(&self, idx: usize) -> Option<[Ptr; 3]> { - self.fetch_ptrs::<6, 3>(idx) - } - #[inline] - pub fn fetch_4_ptrs(&self, idx: usize) -> Option<[Ptr; 4]> { - self.fetch_ptrs::<8, 4>(idx) - } - #[inline] - pub fn expect_2_ptrs(&self, idx: usize) -> [Ptr; 2] { - self.fetch_2_ptrs(idx).expect("Index missing from store") - } - #[inline] - pub fn expect_3_ptrs(&self, idx: usize) -> [Ptr; 3] { - self.fetch_3_ptrs(idx).expect("Index missing from store") - } - #[inline] - pub fn expect_4_ptrs(&self, idx: usize) -> [Ptr; 4] { - self.fetch_4_ptrs(idx).expect("Index missing from store") - } - - #[inline] - pub fn tag(&self, tag: Tag) -> RawPtr { - self.intern_raw_atom(tag.to_field()) - } - - #[inline] - pub fn fetch_tag(&self, ptr: &RawPtr) -> Option { - let idx = ptr.get_atom()?; - let f = self.fetch_f(idx)?; - TagTrait::from_field(f) - } - - pub fn raw_to_ptr(&self, tag: &RawPtr, pay: &RawPtr) -> Option { - let tag = self.fetch_tag(tag)?; - Some(Ptr::new(tag, *pay)) - } - - #[inline] - pub fn num(&self, f: F) -> Ptr { - self.intern_atom(Tag::Expr(Num), f) - } - - #[inline] - pub fn num_u64(&self, u: u64) -> Ptr { - self.intern_atom(Tag::Expr(Num), F::from_u64(u)) - } - - #[inline] - pub fn u64(&self, u: u64) -> Ptr { - self.intern_atom(Tag::Expr(U64), F::from_u64(u)) - } - - #[inline] - pub fn char(&self, c: char) -> Ptr { - self.intern_atom(Tag::Expr(Char), F::from_char(c)) - } - - #[inline] - pub fn comm(&self, hash: F) -> Ptr { - self.intern_atom(Tag::Expr(Comm), hash) - } - - #[inline] - pub fn raw_zero(&self) -> RawPtr { - self.intern_raw_atom(F::ZERO) - } - - #[inline] - pub fn zero(&self, tag: Tag) -> Ptr { - Ptr::new(tag, self.raw_zero()) - } - - pub fn is_zero(&self, ptr: &RawPtr) -> bool { - match ptr { - RawPtr::Atom(idx) => self.fetch_f(*idx) == Some(&F::ZERO), - _ => false, - } - } - - #[inline] - pub fn dummy(&self) -> Ptr { - Ptr::new(Tag::Expr(Nil), self.raw_zero()) - } - - /// Creates an atom pointer from a `ZPtr`, with its hash. Hashing - /// such pointer will result on the same original `ZPtr` - #[inline] - pub fn opaque(&self, z: FWrap) -> RawPtr { - self.intern_raw_atom(z.0) - } - - pub fn intern_string(&self, s: &str) -> Ptr { - if let Some(ptr) = self.string_ptr_cache.get(s) { - *ptr - } else { - let nil_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); - let ptr = s.chars().rev().fold(nil_str, |acc, c| { - let ptrs = [self.char(c), acc]; - self.intern_ptrs::<2>(Tag::Expr(Str), ptrs) - }); - self.string_ptr_cache.insert(s.to_string(), Box::new(ptr)); - self.ptr_string_cache.insert(ptr, s.to_string()); - ptr - } - } - - pub fn fetch_string(&self, ptr: &Ptr) -> Option { - if let Some(str) = self.ptr_string_cache.get(ptr) { - Some(str.to_string()) - } else { - let mut string = String::new(); - let mut ptr = *ptr; - if *ptr.tag() != Tag::Expr(Str) { - return None; - } - loop { - match *ptr.pay() { - RawPtr::Atom(idx) => { - if self.fetch_f(idx)? == &F::ZERO { - self.ptr_string_cache.insert(ptr, string.clone()); - return Some(string); - } else { - return None; - } - } - RawPtr::Hash4(idx) => { - let [car_tag, car, cdr_tag, cdr] = self.fetch_raw_ptrs(idx)?; - assert_eq!(*car_tag, self.tag(Tag::Expr(Char))); - assert_eq!(*cdr_tag, self.tag(Tag::Expr(Str))); - match car { - RawPtr::Atom(idx) => { - let f = self.fetch_f(*idx)?; - string.push(f.to_char().expect("malformed char pointer")); - ptr = Ptr::new(Tag::Expr(Str), *cdr) - } - _ => return None, - } - } - _ => return None, - } - } - } - } - - pub fn intern_symbol_path(&self, path: &[String]) -> Ptr { - let zero_sym = Ptr::new(Tag::Expr(Sym), self.raw_zero()); - path.iter().fold(zero_sym, |acc, s| { - let ptrs = [self.intern_string(s), acc]; - self.intern_ptrs::<2>(Tag::Expr(Sym), ptrs) - }) - } - - pub fn intern_symbol(&self, sym: &Symbol) -> Ptr { - if let Some(ptr) = self.symbol_ptr_cache.get(sym) { - *ptr - } else { - let path_ptr = self.intern_symbol_path(sym.path()); - let sym_ptr = if sym == &lurk_sym("nil") { - Ptr::new(Tag::Expr(Nil), *path_ptr.pay()) - } else if sym.is_keyword() { - Ptr::new(Tag::Expr(Key), *path_ptr.pay()) - } else { - path_ptr - }; - self.symbol_ptr_cache.insert(sym.clone(), Box::new(sym_ptr)); - self.ptr_symbol_cache.insert(sym_ptr, Box::new(sym.clone())); - sym_ptr - } - } - - /// Fetches a symbol path whose interning returned the provided `idx` - fn fetch_symbol_path(&self, mut idx: usize) -> Option> { - let mut path = vec![]; - loop { - let [car_tag, car, cdr_tag, cdr] = self.fetch_raw_ptrs(idx)?; - assert_eq!(*car_tag, self.tag(Tag::Expr(Str))); - assert_eq!(*cdr_tag, self.tag(Tag::Expr(Sym))); - let string = self.fetch_string(&Ptr::new(Tag::Expr(Str), *car))?; - path.push(string); - match cdr { - RawPtr::Atom(idx) => { - if self.fetch_f(*idx)? == &F::ZERO { - path.reverse(); - return Some(path); - } else { - return None; - } - } - RawPtr::Hash4(idx_cdr) => idx = *idx_cdr, - _ => return None, - } - } - } - - pub fn fetch_symbol(&self, ptr: &Ptr) -> Option { - if let Some(sym) = self.ptr_symbol_cache.get(ptr) { - Some(sym.clone()) - } else { - match (ptr.tag(), ptr.pay()) { - (Tag::Expr(Sym), RawPtr::Atom(idx)) => { - if self.fetch_f(*idx)? == &F::ZERO { - let sym = Symbol::root_sym(); - self.ptr_symbol_cache.insert(*ptr, Box::new(sym.clone())); - Some(sym) - } else { - None - } - } - (Tag::Expr(Key), RawPtr::Atom(idx)) => { - if self.fetch_f(*idx)? == &F::ZERO { - let key = Symbol::root_key(); - self.ptr_symbol_cache.insert(*ptr, Box::new(key.clone())); - Some(key) - } else { - None - } - } - (Tag::Expr(Sym | Nil), RawPtr::Hash4(idx)) => { - let path = self.fetch_symbol_path(*idx)?; - let sym = Symbol::sym_from_vec(path); - self.ptr_symbol_cache.insert(*ptr, Box::new(sym.clone())); - Some(sym) - } - (Tag::Expr(Key), RawPtr::Hash4(idx)) => { - let path = self.fetch_symbol_path(*idx)?; - let key = Symbol::key_from_vec(path); - self.ptr_symbol_cache.insert(*ptr, Box::new(key.clone())); - Some(key) - } - _ => None, - } - } - } - - pub fn fetch_sym(&self, ptr: &Ptr) -> Option { - if ptr.tag() == &Tag::Expr(Sym) { - self.fetch_symbol(ptr) - } else { - None - } - } - - pub fn fetch_key(&self, ptr: &Ptr) -> Option { - if ptr.tag() == &Tag::Expr(Key) { - self.fetch_symbol(ptr) - } else { - None - } - } - - #[inline] - pub fn intern_lurk_symbol(&self, name: &str) -> Ptr { - self.intern_symbol(&lurk_sym(name)) - } - - #[inline] - pub fn intern_nil(&self) -> Ptr { - self.intern_lurk_symbol("nil") - } - - #[inline] - pub fn intern_user_symbol(&self, name: &str) -> Ptr { - self.intern_symbol(&user_sym(name)) - } - - #[inline] - pub fn key(&self, name: &str) -> Ptr { - self.intern_symbol(&Symbol::key(&[name.to_string()])) - } - - #[inline] - pub fn add_comm(&self, hash: F, secret: F, payload: Ptr) { - self.comms.insert(FWrap(hash), Box::new((secret, payload))); - } - - #[inline] - pub fn hide(&self, secret: F, payload: Ptr) -> Ptr { - self.comm(self.hide_ptr(secret, payload)) - } - - pub fn hide_and_return_z_payload(&self, secret: F, payload: Ptr) -> (F, ZPtr) { - let z_ptr = self.hash_ptr(&payload); - let hash = self - .poseidon_cache - .hash3(&[secret, z_ptr.tag_field(), *z_ptr.value()]); - self.add_comm(hash, secret, payload); - (hash, z_ptr) - } - - #[inline] - pub fn commit(&self, payload: Ptr) -> Ptr { - self.hide(F::NON_HIDING_COMMITMENT_SECRET, payload) - } - - #[inline] - pub fn open(&self, hash: F) -> Option<&(F, Ptr)> { - self.comms.get(&FWrap(hash)) - } - - pub fn hide_ptr(&self, secret: F, payload: Ptr) -> F { - let hash = self.poseidon_cache.hash3(&[ - secret, - payload.tag().to_field(), - self.hash_raw_ptr(payload.pay()).0, - ]); - self.add_comm(hash, secret, payload); - hash - } - - #[inline] - pub fn cons(&self, car: Ptr, cdr: Ptr) -> Ptr { - let ptrs = [car, cdr]; - self.intern_ptrs::<2>(Tag::Expr(Cons), ptrs) - } - - #[inline] - pub fn intern_fun(&self, arg: Ptr, body: Ptr, env: Ptr) -> Ptr { - let ptrs = [arg, body, env, self.dummy()]; - self.intern_ptrs::<4>(Tag::Expr(Fun), ptrs) - } - - #[inline] - pub fn cont_outermost(&self) -> Ptr { - Ptr::new(Tag::Cont(Outermost), RawPtr::Atom(self.hash8zeros_idx)) - } - - #[inline] - pub fn cont_error(&self) -> Ptr { - Ptr::new(Tag::Cont(ContTag::Error), RawPtr::Atom(self.hash8zeros_idx)) - } - - #[inline] - pub fn cont_terminal(&self) -> Ptr { - Ptr::new(Tag::Cont(Terminal), RawPtr::Atom(self.hash8zeros_idx)) - } - - pub fn car_cdr(&self, ptr: &Ptr) -> Result<(Ptr, Ptr)> { - match ptr.tag() { - Tag::Expr(Nil) => { - let nil = self.intern_nil(); - Ok((nil, nil)) - } - Tag::Expr(Cons) => { - let Some(idx) = ptr.pay().get_hash4() else { - bail!("malformed cons pointer") - }; - match self.fetch_raw_ptrs(idx) { - Some([car_tag, car, cdr_tag, cdr]) => { - let car_ptr = self.raw_to_ptr(car_tag, car).context("Not a pointer")?; - let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr).context("Not a pointer")?; - Ok((car_ptr, cdr_ptr)) - } - None => bail!("car/cdr not found"), - } - } - Tag::Expr(Str) => { - if self.is_zero(ptr.pay()) { - let nil_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); - Ok((self.intern_nil(), nil_str)) - } else { - let Some(idx) = ptr.pay().get_hash4() else { - bail!("malformed str pointer") - }; - match self.fetch_raw_ptrs(idx) { - Some([car_tag, car, cdr_tag, cdr]) => { - let car_ptr = self.raw_to_ptr(car_tag, car).context("Not a pointer")?; - let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr).context("Not a pointer")?; - Ok((car_ptr, cdr_ptr)) - } - None => bail!("car/cdr not found"), - } - } - } - _ => bail!("invalid pointer to extract car/cdr from"), - } - } - - /// Interns a sequence of pointers as a cons-list. The terminating element - /// defaults to `nil` if `last` is `None` - fn intern_list(&self, elts: Vec, last: Option) -> Ptr { - elts.into_iter() - .rev() - .fold(last.unwrap_or_else(|| self.intern_nil()), |acc, elt| { - self.cons(acc, elt) - }) - } - - /// Interns a sequence of pointers as a proper (`nil`-terminated) cons-list - #[inline] - pub fn list(&self, elts: Vec) -> Ptr { - self.intern_list(elts, None) - } - - /// Interns a sequence of pointers as an improper cons-list whose last - /// element is `last` - #[inline] - pub fn improper_list(&self, elts: Vec, last: Ptr) -> Ptr { - self.intern_list(elts, Some(last)) - } - - /// Fetches a cons list that was interned. If the list is improper, the second - /// element of the returned pair will carry the improper terminating value - pub fn fetch_list(&self, ptr: &Ptr) -> Option<(Vec, Option)> { - if *ptr == self.intern_nil() { - return Some((vec![], None)); - } - match (ptr.tag(), ptr.pay()) { - (Tag::Expr(Nil), _) => panic!("Malformed nil expression"), - (Tag::Expr(Cons), RawPtr::Hash4(mut idx)) => { - let mut list = vec![]; - let mut last = None; - while let Some([car_tag, car, cdr_tag, cdr]) = self.fetch_raw_ptrs(idx) { - let car_ptr = self.raw_to_ptr(car_tag, car)?; - let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr)?; - list.push(car_ptr); - match cdr_ptr.tag() { - Tag::Expr(Nil) => break, - Tag::Expr(Cons) => { - idx = cdr.get_hash4()?; - } - _ => { - last = Some(cdr_ptr); - break; - } - } - } - Some((list, last)) - } - _ => None, - } - } - - pub fn intern_syntax(&self, syn: Syntax) -> Ptr { - match syn { - Syntax::Num(_, x) => self.num(x.into_scalar()), - Syntax::UInt(_, x) => self.u64(x.into()), - Syntax::Char(_, x) => self.char(x), - Syntax::Symbol(_, x) => self.intern_symbol(&x), - Syntax::String(_, x) => self.intern_string(&x), - Syntax::Quote(_, x) => self.list(vec![ - self.intern_symbol(&lurk_sym("quote")), - self.intern_syntax(*x), - ]), - Syntax::List(_, xs) => { - self.list(xs.into_iter().map(|x| self.intern_syntax(x)).collect()) - } - Syntax::Improper(_, xs, y) => self.improper_list( - xs.into_iter().map(|x| self.intern_syntax(x)).collect(), - self.intern_syntax(*y), - ), - } - } - - pub fn read(&self, state: Rc>, input: &str) -> Result { - match preceded( - syntax::parse_space, - syntax::parse_syntax(state, false, false), - ) - .parse(Span::new(input)) - { - Ok((_, x)) => Ok(self.intern_syntax(x)), - Err(e) => bail!("{}", e), - } - } - - pub fn read_maybe_meta<'a>( - &self, - state: Rc>, - input: &'a str, - ) -> Result<(usize, Span<'a>, Ptr, bool), Error> { - match preceded(syntax::parse_space, syntax::parse_maybe_meta(state, false)) - .parse(input.into()) - { - Ok((i, Some((is_meta, x)))) => { - let from_offset = x - .get_pos() - .get_from_offset() - .expect("Parsed syntax should have its Pos set"); - Ok((from_offset, i, self.intern_syntax(x), is_meta)) - } - Ok((_, None)) => Err(Error::NoInput), - Err(e) => Err(Error::Syntax(format!("{}", e))), - } - } - - #[inline] - pub fn read_with_default_state(&self, input: &str) -> Result { - self.read(State::init_lurk_state().rccell(), input) - } - - /// Recursively hashes the children of a `Ptr` in order to obtain its - /// corresponding `ZPtr`. While traversing a `Ptr` tree, it consults the - /// cache of `Ptr`s that have already been hydrated and also populates this - /// cache for the new `Ptr`s. - /// - /// Warning: without cache hits, this function might blow up Rust's recursion - /// depth limit. This limitation is circumvented by calling `hydrate_z_cache` - /// beforehand or by using `hash_ptr` instead, which is slightly slower. - fn hash_raw_ptr_unsafe(&self, ptr: &RawPtr) -> FWrap { - match ptr { - RawPtr::Atom(idx) => FWrap(*self.expect_f(*idx)), - RawPtr::Hash3(idx) => { - if let Some(z) = self.z_cache.get(ptr) { - *z - } else { - let children_ptrs = self.expect_raw_ptrs::<3>(*idx); - let mut children_zs = [F::ZERO; 3]; - for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; - } - let z = FWrap(self.poseidon_cache.hash3(&children_zs)); - self.z_cache.insert(*ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(*ptr)); - z - } - } - RawPtr::Hash4(idx) => { - if let Some(z) = self.z_cache.get(ptr) { - *z - } else { - let children_ptrs = self.expect_raw_ptrs::<4>(*idx); - let mut children_zs = [F::ZERO; 4]; - for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; - } - let z = FWrap(self.poseidon_cache.hash4(&children_zs)); - self.z_cache.insert(*ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(*ptr)); - z - } - } - RawPtr::Hash6(idx) => { - if let Some(z) = self.z_cache.get(ptr) { - *z - } else { - let children_ptrs = self.expect_raw_ptrs::<6>(*idx); - let mut children_zs = [F::ZERO; 6]; - for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; - } - let z = FWrap(self.poseidon_cache.hash6(&children_zs)); - self.z_cache.insert(*ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(*ptr)); - z - } - } - RawPtr::Hash8(idx) => { - if let Some(z) = self.z_cache.get(ptr) { - *z - } else { - let children_ptrs = self.expect_raw_ptrs::<8>(*idx); - let mut children_zs = [F::ZERO; 8]; - for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; - } - let z = FWrap(self.poseidon_cache.hash8(&children_zs)); - self.z_cache.insert(*ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(*ptr)); - z - } - } - } - } - - /// Hashes pointers in parallel, consuming chunks of length 256, which is a - /// reasonably safe limit. The danger of longer chunks is that the rightmost - /// pointers are the ones which are more likely to reach the recursion depth - /// limit in `hash_ptr_unsafe`. So we move in smaller chunks from left to - /// right, populating the `z_cache`, which can rescue `hash_ptr_unsafe` from - /// dangerously deep recursions - fn hydrate_z_cache_with_ptrs(&self, ptrs: &[&RawPtr]) { - ptrs.chunks(256).for_each(|chunk| { - chunk.par_iter().for_each(|ptr| { - self.hash_raw_ptr_unsafe(ptr); - }); - }); - } - - /// Hashes enqueued `Ptr` trees from the bottom to the top, avoiding deep - /// recursions in `hash_ptr`. Resets the `dehydrated` queue afterwards. - pub fn hydrate_z_cache(&self) { - self.hydrate_z_cache_with_ptrs(&self.dehydrated.load().iter().collect::>()); - self.dehydrated.swap(Arc::new(FrozenVec::default())); - } - - /// Whether the length of the dehydrated queue is within the safe limit. - /// Note: these values are experimental and may be machine dependant. - #[inline] - fn is_below_safe_threshold(&self) -> bool { - if cfg!(debug_assertions) { - // not release mode - self.dehydrated.load().len() < 443 - } else { - // release mode - self.dehydrated.load().len() < 2497 - } - } - - /// Safe version of `hash_ptr_unsafe` that doesn't hit a stack overflow by - /// precomputing the pointers that need to be hashed in order to hash the - /// provided `ptr` - pub fn hash_raw_ptr(&self, ptr: &RawPtr) -> FWrap { - if self.is_below_safe_threshold() { - // just run `hash_ptr_unsafe` for extra speed when the dehydrated - // queue is small enough - return self.hash_raw_ptr_unsafe(ptr); - } - let mut ptrs: IndexSet<&RawPtr> = IndexSet::default(); - let mut stack = vec![ptr]; - macro_rules! feed_loop { - ($x:expr) => { - if $x.is_hash() { - if self.z_cache.get($x).is_none() { - if ptrs.insert($x) { - stack.push($x); - } - } - } - }; - } - while let Some(ptr) = stack.pop() { - match ptr { - RawPtr::Atom(..) => (), - RawPtr::Hash3(idx) => { - let ptrs = self.expect_raw_ptrs::<3>(*idx); - for ptr in ptrs { - feed_loop!(ptr) - } - } - RawPtr::Hash4(idx) => { - let ptrs = self.expect_raw_ptrs::<4>(*idx); - for ptr in ptrs { - feed_loop!(ptr) - } - } - RawPtr::Hash6(idx) => { - let ptrs = self.expect_raw_ptrs::<6>(*idx); - for ptr in ptrs { - feed_loop!(ptr) - } - } - RawPtr::Hash8(idx) => { - let ptrs = self.expect_raw_ptrs::<8>(*idx); - for ptr in ptrs { - feed_loop!(ptr) - } - } - } - } - ptrs.reverse(); - self.hydrate_z_cache_with_ptrs(&ptrs.into_iter().collect::>()); - // Now it's okay to call `hash_ptr_unsafe` - self.hash_raw_ptr_unsafe(ptr) - } - - pub fn hash_ptr(&self, ptr: &Ptr) -> ZPtr { - ZPtr::from_parts(*ptr.tag(), self.hash_raw_ptr(ptr.pay()).0) - } - - /// Constructs a vector of scalars that correspond to tags and hashes computed - /// from a slice of `Ptr`s turned into `ZPtr`s - pub fn to_scalar_vector(&self, ptrs: &[Ptr]) -> Vec { - ptrs.iter() - .fold(Vec::with_capacity(2 * ptrs.len()), |mut acc, ptr| { - let tag = ptr.tag().to_field(); - let pay = self.hash_raw_ptr(ptr.pay()).0; - acc.push(tag); - acc.push(pay); - acc - }) - } - - pub fn to_scalar_vector_raw(&self, ptrs: &[RawPtr]) -> Vec { - ptrs.iter().map(|ptr| self.hash_raw_ptr(ptr).0).collect() - } - - /// Equality of the content-addressed versions of two pointers - #[inline] - pub fn raw_ptr_eq(&self, a: &RawPtr, b: &RawPtr) -> bool { - self.hash_raw_ptr(a) == self.hash_raw_ptr(b) - } - - #[inline] - pub fn ptr_eq(&self, a: &Ptr, b: &Ptr) -> bool { - self.hash_ptr(a) == self.hash_ptr(b) - } - - /// Attempts to recover the `Ptr` that corresponds to `z_ptr` from - /// `inverse_z_cache`. If the mapping is not there, returns an atom pointer - /// with the same tag and value - #[inline] - pub fn to_raw_ptr(&self, z: &FWrap) -> RawPtr { - self.inverse_z_cache - .get(z) - .cloned() - .unwrap_or_else(|| self.opaque(*z)) - } - - #[inline] - pub fn to_ptr(&self, z_ptr: &ZPtr) -> Ptr { - Ptr::new(*z_ptr.tag(), self.to_raw_ptr(&FWrap(*z_ptr.value()))) - } -} - -impl Ptr { - pub fn fmt_to_string(&self, store: &Store, state: &State) -> String { - match self.tag() { - Tag::Expr(t) => match t { - Nil => { - if let Some(sym) = store.fetch_symbol(self) { - state.fmt_to_string(&sym.into()) - } else { - "".into() - } - } - Sym => { - if let Some(sym) = store.fetch_sym(self) { - state.fmt_to_string(&sym.into()) - } else { - "".into() - } - } - Key => { - if let Some(key) = store.fetch_key(self) { - state.fmt_to_string(&key.into()) - } else { - "".into() - } - } - Str => { - if let Some(str) = store.fetch_string(self) { - format!("\"{str}\"") - } else { - "".into() - } - } - Char => { - if let Some(c) = self - .pay() - .get_atom() - .map(|idx| store.expect_f(idx)) - .and_then(F::to_char) - { - format!("\'{c}\'") - } else { - "".into() - } - } - Cons => { - if let Some((list, non_nil)) = store.fetch_list(self) { - let list = list - .iter() - .map(|p| p.fmt_to_string(store, state)) - .collect::>(); - if let Some(non_nil) = non_nil { - format!( - "({} . {})", - list.join(" "), - non_nil.fmt_to_string(store, state) - ) - } else { - format!("({})", list.join(" ")) - } - } else { - "".into() - } - } - Num => { - if let Some(f) = self.pay().get_atom().map(|idx| store.expect_f(idx)) { - if let Some(u) = f.to_u64() { - u.to_string() - } else { - format!("0x{}", f.hex_digits()) - } - } else { - "".into() - } - } - U64 => { - if let Some(u) = self - .pay() - .get_atom() - .map(|idx| store.expect_f(idx)) - .and_then(F::to_u64) - { - format!("{u}u64") - } else { - "".into() - } - } - Fun => match self.pay().get_hash8() { - None => "".into(), - Some(idx) => { - if let Some([vars, body, _, _]) = store.fetch_ptrs::<8, 4>(idx) { - match vars.tag() { - Tag::Expr(Nil) => { - format!("", body.fmt_to_string(store, state)) - } - Tag::Expr(Cons) => { - format!( - "", - vars.fmt_to_string(store, state), - body.fmt_to_string(store, state) - ) - } - _ => "".into(), - } - } else { - "".into() - } - } - }, - Thunk => match self.pay().get_hash4() { - None => "".into(), - Some(idx) => { - if let Some([val, cont]) = store.fetch_ptrs::<4, 2>(idx) { - format!( - "Thunk{{ value: {} => cont: {} }}", - val.fmt_to_string(store, state), - cont.fmt_to_string(store, state) - ) - } else { - "".into() - } - } - }, - Comm => match self.pay().get_atom() { - Some(idx) => { - let f = store.expect_f(idx); - if store.comms.get(&FWrap(*f)).is_some() { - format!("(comm 0x{})", f.hex_digits()) - } else { - format!("", f.hex_digits()) - } - } - None => "".into(), - }, - Cproc => match self.pay().get_hash4() { - None => "".into(), - Some(idx) => { - if let Some([cproc_name, args]) = store.fetch_ptrs::<4, 2>(idx) { - format!( - "", - cproc_name.fmt_to_string(store, state), - args.fmt_to_string(store, state) - ) - } else { - "".into() - } - } - }, - }, - Tag::Cont(t) => match t { - Outermost => "Outermost".into(), - Dummy => "Dummy".into(), - ContTag::Error => "Error".into(), - Terminal => "Terminal".into(), - Call0 => self.fmt_cont2_to_string("Call0", "saved_env", store, state), - Call => { - self.fmt_cont3_to_string("Call", ("unevaled_arg", "saved_env"), store, state) - } - Call2 => self.fmt_cont3_to_string("Call2", ("function", "saved_env"), store, state), - Tail => self.fmt_cont2_to_string("Tail", "saved_env", store, state), - Lookup => self.fmt_cont2_to_string("Lookup", "saved_env", store, state), - Unop => self.fmt_cont2_to_string("Unop", "saved_env", store, state), - Binop => self.fmt_cont4_to_string( - "Binop", - ("operator", "saved_env", "unevaled_args"), - store, - state, - ), - Binop2 => { - self.fmt_cont3_to_string("Binop2", ("operator", "evaled_arg"), store, state) - } - If => self.fmt_cont2_to_string("If", "unevaled_args", store, state), - Let => self.fmt_cont4_to_string("Let", ("var", "saved_env", "body"), store, state), - LetRec => { - self.fmt_cont4_to_string("LetRec", ("var", "saved_env", "body"), store, state) - } - Emit => "Emit ".into(), - ContTag::Cproc => self.fmt_cont4_to_string( - "Cproc", - ("name", "unevaled_args", "evaled_args"), - store, - state, - ), - }, - Tag::Op1(op) => op.to_string(), - Tag::Op2(op) => op.to_string(), - } - } - - fn fmt_cont2_to_string( - &self, - name: &str, - field: &str, - store: &Store, - state: &State, - ) -> String { - match self.pay().get_hash8() { - None => format!(""), - Some(idx) => { - if let Some([a, cont, _, _]) = store.fetch_ptrs::<8, 4>(idx) { - format!( - "{name}{{ {field}: {}, continuation: {} }}", - a.fmt_to_string(store, state), - cont.fmt_to_string(store, state) - ) - } else { - format!("") - } - } - } - } - - fn fmt_cont3_to_string( - &self, - name: &str, - fields: (&str, &str), - store: &Store, - state: &State, - ) -> String { - match self.pay().get_hash8() { - None => format!(""), - Some(idx) => { - if let Some([a, b, cont, _]) = store.fetch_ptrs::<8, 4>(idx) { - let (fa, fb) = fields; - format!( - "{name}{{ {fa}: {}, {fb}: {}, continuation: {} }}", - a.fmt_to_string(store, state), - b.fmt_to_string(store, state), - cont.fmt_to_string(store, state) - ) - } else { - format!("") - } - } - } - } - - fn fmt_cont4_to_string( - &self, - name: &str, - fields: (&str, &str, &str), - store: &Store, - state: &State, - ) -> String { - match self.pay().get_hash8() { - None => format!(""), - Some(idx) => { - if let Some([a, b, c, cont]) = store.fetch_ptrs::<8, 4>(idx) { - let (fa, fb, fc) = fields; - format!( - "{name}{{ {fa}: {}, {fb}: {}, {fc}: {}, continuation: {} }}", - a.fmt_to_string(store, state), - b.fmt_to_string(store, state), - c.fmt_to_string(store, state), - cont.fmt_to_string(store, state) - ) - } else { - format!("") - } - } - } - } -} diff --git a/src/lem/store.rs b/src/lem/store.rs index acb504d97a..ba38240235 100644 --- a/src/lem/store.rs +++ b/src/lem/store.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; use arc_swap::ArcSwap; use bellpepper::util_cs::witness_cs::SizedWitness; use elsa::{ @@ -24,34 +24,18 @@ use crate::{ Tail, Terminal, Unop, }, tag::ExprTag::{Char, Comm, Cons, Cproc, Fun, Key, Nil, Num, Str, Sym, Thunk, U64}, + tag::Tag as TagTrait, }; -use super::pointers::{Ptr, ZPtr}; - -/// The `Store` is a crucial part of Lurk's implementation and tries to be a -/// vesatile data structure for many parts of Lurk's data pipeline. -/// -/// It holds Lurk data structured as trees of `Ptr`s. When a `Ptr` has children, -/// we store them in the `IndexSet`s available: `tuple2`, `tuple3` or `tuple4`. -/// These data structures speed up LEM interpretation because lookups by indices -/// are fast. -/// -/// The `Store` provides an infra to speed up interning strings and symbols. This -/// data is saved in `string_ptr_cache` and `symbol_ptr_cache`. -/// -/// There's also a process that we call "hydration", in which we use Poseidon -/// hashes to compute the (stable) hash of the children of a pointer. These hashes -/// are necessary when we want to create Lurk proofs because the circuit consumes -/// elements of the `LurkField`, not (unstable) indices of `IndexSet`s. -/// -/// Lastly, we have a `HashMap` to hold committed data, which can be retrieved by -/// the resulting commitment hash. +use super::pointers::{Ptr, RawPtr, ZPtr}; + #[derive(Debug)] pub struct Store { f_elts: FrozenIndexSet>>, - tuple2: FrozenIndexSet>, - tuple3: FrozenIndexSet>, - tuple4: FrozenIndexSet>, + hash3: FrozenIndexSet>, + hash4: FrozenIndexSet>, + hash6: FrozenIndexSet>, + hash8: FrozenIndexSet>, string_ptr_cache: FrozenMap>, symbol_ptr_cache: FrozenMap>, @@ -59,14 +43,14 @@ pub struct Store { ptr_string_cache: FrozenMap, ptr_symbol_cache: FrozenMap>, + comms: FrozenMap, Box<(F, Ptr)>>, // hash -> (secret, src) + pub poseidon_cache: PoseidonCache, pub inverse_poseidon_cache: InversePoseidonCache, - dehydrated: ArcSwap>>, - z_cache: FrozenMap>>, - inverse_z_cache: FrozenMap, Box>, - - comms: FrozenMap, Box<(F, Ptr)>>, // hash -> (secret, src) + dehydrated: ArcSwap>>, + z_cache: FrozenMap>>, + inverse_z_cache: FrozenMap, Box>, // cached indices for the hashes of 3, 4, 6 and 8 padded zeros pub hash3zeros_idx: usize, @@ -91,19 +75,20 @@ impl Default for Store { Self { f_elts, - tuple2: Default::default(), - tuple3: Default::default(), - tuple4: Default::default(), + hash3: Default::default(), + hash4: Default::default(), + hash6: Default::default(), + hash8: Default::default(), string_ptr_cache: Default::default(), symbol_ptr_cache: Default::default(), ptr_string_cache: Default::default(), ptr_symbol_cache: Default::default(), + comms: Default::default(), poseidon_cache, inverse_poseidon_cache: Default::default(), dehydrated: Default::default(), z_cache: Default::default(), inverse_z_cache: Default::default(), - comms: Default::default(), hash3zeros_idx, hash4zeros_idx, hash6zeros_idx, @@ -166,11 +151,33 @@ impl Store { self.f_elts.insert_probe(Box::new(FWrap(f))) } - /// Creates an atom `Ptr` which points to a cached element of the finite + /// Creates an atom `RawPtr` which points to a cached element of the finite /// field `F` + pub fn intern_raw_atom(&self, f: F) -> RawPtr { + let (idx, _) = self.intern_f(f); + RawPtr::Atom(idx) + } + pub fn intern_atom(&self, tag: Tag, f: F) -> Ptr { - let (idx, inserted) = self.intern_f(f); - let ptr = Ptr::Atom(tag, idx); + Ptr::new(tag, self.intern_raw_atom(f)) + } + + /// Creates a `RawPtr` that's a parent of `N` children + pub fn intern_raw_ptrs(&self, ptrs: [RawPtr; N]) -> RawPtr { + macro_rules! intern { + ($Hash:ident, $hash:ident, $n:expr) => {{ + let ptrs = unsafe { std::mem::transmute::<&[RawPtr; N], &[RawPtr; $n]>(&ptrs) }; + let (idx, inserted) = self.$hash.insert_probe(Box::new(*ptrs)); + (RawPtr::$Hash(idx), inserted) + }}; + } + let (ptr, inserted) = match N { + 3 => intern!(Hash3, hash3, 3), + 4 => intern!(Hash4, hash4, 4), + 6 => intern!(Hash6, hash6, 6), + 8 => intern!(Hash8, hash8, 8), + _ => unimplemented!(), + }; if inserted { // this is for `hydrate_z_cache` self.dehydrated.load().push(Box::new(ptr)); @@ -178,72 +185,161 @@ impl Store { ptr } - /// Similar to `intern_atom` but doesn't add the resulting pointer to + /// Similar to `intern_raw_ptrs` but doesn't add the resulting pointer to /// `dehydrated`. This function is used when converting a `ZStore` to a /// `Store`. - pub fn intern_atom_hydrated(&self, tag: Tag, f: F, z: ZPtr) -> Ptr { - let ptr = Ptr::Atom(tag, self.intern_f(f).0); + pub fn intern_raw_ptrs_hydrated( + &self, + ptrs: [RawPtr; N], + z: FWrap, + ) -> RawPtr { + macro_rules! intern { + ($Hash:ident, $hash:ident, $n:expr) => {{ + let ptrs = unsafe { std::mem::transmute::<&[RawPtr; N], &[RawPtr; $n]>(&ptrs) }; + let (idx, _) = self.$hash.insert_probe(Box::new(*ptrs)); + RawPtr::$Hash(idx) + }}; + } + let ptr = match N { + 3 => intern!(Hash3, hash3, 3), + 4 => intern!(Hash4, hash4, 4), + 6 => intern!(Hash6, hash6, 6), + 8 => intern!(Hash8, hash8, 8), + _ => unimplemented!(), + }; self.z_cache.insert(ptr, Box::new(z)); self.inverse_z_cache.insert(z, Box::new(ptr)); ptr } - /// Creates a `Ptr` that's a parent of two children - pub fn intern_2_ptrs(&self, tag: Tag, a: Ptr, b: Ptr) -> Ptr { - let (idx, inserted) = self.tuple2.insert_probe(Box::new((a, b))); - let ptr = Ptr::Tuple2(tag, idx); - if inserted { - // this is for `hydrate_z_cache` - self.dehydrated.load().push(Box::new(ptr)); + /// Creates a `Ptr` that's a parent of `N` children + pub fn intern_ptrs(&self, tag: Tag, ptrs: [Ptr; N]) -> Ptr { + macro_rules! intern { + ($n:expr) => {{ + let mut raw_ptrs = [self.raw_zero(); $n * 2]; + for i in 0..$n { + raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); + raw_ptrs[2 * i + 1] = *ptrs[i].pay(); + } + self.intern_raw_ptrs::<{ $n * 2 }>(raw_ptrs) + }}; } - ptr + let pay = match N { + 2 => intern!(2), + 3 => intern!(3), + 4 => intern!(4), + _ => unimplemented!(), + }; + Ptr::new(tag, pay) } - /// Similar to `intern_2_ptrs` but doesn't add the resulting pointer to + /// Similar to `intern_ptrs` but doesn't add the resulting pointer to /// `dehydrated`. This function is used when converting a `ZStore` to a /// `Store`. - pub fn intern_2_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, z: ZPtr) -> Ptr { - let ptr = Ptr::Tuple2(tag, self.tuple2.insert_probe(Box::new((a, b))).0); - self.z_cache.insert(ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(ptr)); - ptr + pub fn intern_ptrs_hydrated( + &self, + tag: Tag, + ptrs: [Ptr; N], + z: FWrap, + ) -> Ptr { + macro_rules! intern { + ($n:expr) => {{ + let mut raw_ptrs = [self.raw_zero(); $n * 2]; + for i in 0..$n { + raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); + raw_ptrs[2 * i + 1] = *ptrs[i].pay(); + } + self.intern_raw_ptrs_hydrated::<{ $n * 2 }>(raw_ptrs, z) + }}; + } + let pay = match N { + 2 => intern!(2), + 3 => intern!(3), + 4 => intern!(4), + _ => unimplemented!(), + }; + Ptr::new(tag, pay) } - /// Creates a `Ptr` that's a parent of three children - pub fn intern_3_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr) -> Ptr { - let (idx, inserted) = self.tuple3.insert_probe(Box::new((a, b, c))); - let ptr = Ptr::Tuple3(tag, idx); - if inserted { - // this is for `hydrate_z_cache` - self.dehydrated.load().push(Box::new(ptr)); - } - ptr + #[inline] + pub fn fetch_f(&self, idx: usize) -> Option<&F> { + self.f_elts.get_index(idx).map(|fw| &fw.0) } - /// Similar to `intern_3_ptrs` but doesn't add the resulting pointer to - /// `dehydrated`. This function is used when converting a `ZStore` to a - /// `Store`. - pub fn intern_3_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, z: ZPtr) -> Ptr { - let ptr = Ptr::Tuple3(tag, self.tuple3.insert_probe(Box::new((a, b, c))).0); - self.z_cache.insert(ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(ptr)); - ptr + #[inline] + pub fn fetch_raw_ptrs(&self, idx: usize) -> Option<&[RawPtr; N]> { + macro_rules! fetch { + ($hash:ident, $n:expr) => {{ + let ptrs = self.$hash.get_index(idx)?; + let ptrs = unsafe { std::mem::transmute::<&[RawPtr; $n], &[RawPtr; N]>(ptrs) }; + Some(ptrs) + }}; + } + match N { + 3 => fetch!(hash3, 3), + 4 => fetch!(hash4, 4), + 6 => fetch!(hash6, 6), + 8 => fetch!(hash8, 8), + _ => unimplemented!(), + } } - /// Creates a `Ptr` that's a parent of four children - pub fn intern_4_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, d: Ptr) -> Ptr { - let (idx, inserted) = self.tuple4.insert_probe(Box::new((a, b, c, d))); - let ptr = Ptr::Tuple4(tag, idx); - if inserted { - // this is for `hydrate_z_cache` - self.dehydrated.load().push(Box::new(ptr)); + #[inline] + pub fn fetch_ptrs(&self, idx: usize) -> Option<[Ptr; P]> { + assert_eq!(P * 2, N); + let raw_ptrs = self + .fetch_raw_ptrs::(idx) + .expect("Index missing from store"); + let mut ptrs = [self.dummy(); P]; + for i in 0..P { + let tag = self.fetch_tag(&raw_ptrs[2 * i])?; + ptrs[i] = Ptr::new(tag, raw_ptrs[2 * i + 1]) } - ptr + Some(ptrs) } - /// Similar to `intern_4_ptrs` but doesn't add the resulting pointer to - /// `dehydrated`. This function is used when converting a `ZStore` to a - /// `Store`. + #[inline] + pub fn expect_f(&self, idx: usize) -> &F { + self.fetch_f(idx).expect("Index missing from f_elts") + } + + #[inline] + pub fn expect_raw_ptrs(&self, idx: usize) -> &[RawPtr; N] { + self.fetch_raw_ptrs::(idx) + .expect("Index missing from store") + } + + #[inline] + pub fn expect_ptrs(&self, idx: usize) -> [Ptr; P] { + self.fetch_ptrs::(idx) + .expect("Index missing from store") + } + + // TODO remove these functions + pub fn intern_atom_hydrated(&self, tag: Tag, f: F, _: ZPtr) -> Ptr { + Ptr::new(tag, self.intern_raw_atom(f)) + } + #[inline] + pub fn intern_2_ptrs(&self, tag: Tag, a: Ptr, b: Ptr) -> Ptr { + self.intern_ptrs::<2>(tag, [a, b]) + } + #[inline] + pub fn intern_3_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr) -> Ptr { + self.intern_ptrs::<3>(tag, [a, b, c]) + } + #[inline] + pub fn intern_4_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, d: Ptr) -> Ptr { + self.intern_ptrs::<4>(tag, [a, b, c, d]) + } + #[inline] + pub fn intern_2_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, z: ZPtr) -> Ptr { + self.intern_ptrs_hydrated::<2>(tag, [a, b], FWrap(*z.value())) + } + #[inline] + pub fn intern_3_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, z: ZPtr) -> Ptr { + self.intern_ptrs_hydrated::<3>(tag, [a, b, c], FWrap(*z.value())) + } + #[inline] pub fn intern_4_ptrs_hydrated( &self, tag: Tag, @@ -253,30 +349,48 @@ impl Store { d: Ptr, z: ZPtr, ) -> Ptr { - let ptr = Ptr::Tuple4(tag, self.tuple4.insert_probe(Box::new((a, b, c, d))).0); - self.z_cache.insert(ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(ptr)); - ptr + self.intern_ptrs_hydrated::<4>(tag, [a, b, c, d], FWrap(*z.value())) } - #[inline] - pub fn fetch_f(&self, idx: usize) -> Option<&F> { - self.f_elts.get_index(idx).map(|fw| &fw.0) + pub fn fetch_2_ptrs(&self, idx: usize) -> Option<[Ptr; 2]> { + self.fetch_ptrs::<4, 2>(idx) + } + #[inline] + pub fn fetch_3_ptrs(&self, idx: usize) -> Option<[Ptr; 3]> { + self.fetch_ptrs::<6, 3>(idx) } - #[inline] - pub fn fetch_2_ptrs(&self, idx: usize) -> Option<&(Ptr, Ptr)> { - self.tuple2.get_index(idx) + pub fn fetch_4_ptrs(&self, idx: usize) -> Option<[Ptr; 4]> { + self.fetch_ptrs::<8, 4>(idx) + } + #[inline] + pub fn expect_2_ptrs(&self, idx: usize) -> [Ptr; 2] { + self.fetch_2_ptrs(idx).expect("Index missing from store") + } + #[inline] + pub fn expect_3_ptrs(&self, idx: usize) -> [Ptr; 3] { + self.fetch_3_ptrs(idx).expect("Index missing from store") + } + #[inline] + pub fn expect_4_ptrs(&self, idx: usize) -> [Ptr; 4] { + self.fetch_4_ptrs(idx).expect("Index missing from store") } #[inline] - pub fn fetch_3_ptrs(&self, idx: usize) -> Option<&(Ptr, Ptr, Ptr)> { - self.tuple3.get_index(idx) + pub fn tag(&self, tag: Tag) -> RawPtr { + self.intern_raw_atom(tag.to_field()) } #[inline] - pub fn fetch_4_ptrs(&self, idx: usize) -> Option<&(Ptr, Ptr, Ptr, Ptr)> { - self.tuple4.get_index(idx) + pub fn fetch_tag(&self, ptr: &RawPtr) -> Option { + let idx = ptr.get_atom()?; + let f = self.fetch_f(idx)?; + TagTrait::from_field(f) + } + + pub fn raw_to_ptr(&self, tag: &RawPtr, pay: &RawPtr) -> Option { + let tag = self.fetch_tag(tag)?; + Some(Ptr::new(tag, *pay)) } #[inline] @@ -304,37 +418,43 @@ impl Store { self.intern_atom(Tag::Expr(Comm), hash) } + #[inline] + pub fn raw_zero(&self) -> RawPtr { + self.intern_raw_atom(F::ZERO) + } + #[inline] pub fn zero(&self, tag: Tag) -> Ptr { - self.intern_atom(tag, F::ZERO) + Ptr::new(tag, self.raw_zero()) } - pub fn is_zero(&self, ptr: &Ptr) -> bool { + pub fn is_zero(&self, ptr: &RawPtr) -> bool { match ptr { - Ptr::Atom(_, idx) => self.fetch_f(*idx) == Some(&F::ZERO), + RawPtr::Atom(idx) => self.fetch_f(*idx) == Some(&F::ZERO), _ => false, } } #[inline] pub fn dummy(&self) -> Ptr { - self.zero(Tag::Expr(Nil)) + Ptr::new(Tag::Expr(Nil), self.raw_zero()) } - /// Creates an atom pointer from a `ZPtr`, with its tag and hash. Thus hashing + /// Creates an atom pointer from a `ZPtr`, with its hash. Hashing /// such pointer will result on the same original `ZPtr` #[inline] - pub fn opaque(&self, z_ptr: ZPtr) -> Ptr { - let crate::z_data::z_ptr::ZPtr(t, h) = z_ptr; - self.intern_atom(t, h) + pub fn opaque(&self, z: FWrap) -> RawPtr { + self.intern_raw_atom(z.0) } pub fn intern_string(&self, s: &str) -> Ptr { if let Some(ptr) = self.string_ptr_cache.get(s) { *ptr } else { - let ptr = s.chars().rev().fold(self.zero(Tag::Expr(Str)), |acc, c| { - self.intern_2_ptrs(Tag::Expr(Str), self.char(c), acc) + let nil_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); + let ptr = s.chars().rev().fold(nil_str, |acc, c| { + let ptrs = [self.char(c), acc]; + self.intern_ptrs::<2>(Tag::Expr(Str), ptrs) }); self.string_ptr_cache.insert(s.to_string(), Box::new(ptr)); self.ptr_string_cache.insert(ptr, s.to_string()); @@ -348,9 +468,12 @@ impl Store { } else { let mut string = String::new(); let mut ptr = *ptr; + if *ptr.tag() != Tag::Expr(Str) { + return None; + } loop { - match ptr { - Ptr::Atom(Tag::Expr(Str), idx) => { + match *ptr.pay() { + RawPtr::Atom(idx) => { if self.fetch_f(idx)? == &F::ZERO { self.ptr_string_cache.insert(ptr, string.clone()); return Some(string); @@ -358,13 +481,15 @@ impl Store { return None; } } - Ptr::Tuple2(Tag::Expr(Str), idx) => { - let (car, cdr) = self.fetch_2_ptrs(idx)?; + RawPtr::Hash4(idx) => { + let [car_tag, car, cdr_tag, cdr] = self.fetch_raw_ptrs(idx)?; + assert_eq!(*car_tag, self.tag(Tag::Expr(Char))); + assert_eq!(*cdr_tag, self.tag(Tag::Expr(Str))); match car { - Ptr::Atom(Tag::Expr(Char), idx) => { + RawPtr::Atom(idx) => { let f = self.fetch_f(*idx)?; string.push(f.to_char().expect("malformed char pointer")); - ptr = *cdr + ptr = Ptr::new(Tag::Expr(Str), *cdr) } _ => return None, } @@ -376,9 +501,10 @@ impl Store { } pub fn intern_symbol_path(&self, path: &[String]) -> Ptr { - path.iter().fold(self.zero(Tag::Expr(Sym)), |acc, s| { - let s_ptr = self.intern_string(s); - self.intern_2_ptrs(Tag::Expr(Sym), s_ptr, acc) + let zero_sym = Ptr::new(Tag::Expr(Sym), self.raw_zero()); + path.iter().fold(zero_sym, |acc, s| { + let ptrs = [self.intern_string(s), acc]; + self.intern_ptrs::<2>(Tag::Expr(Sym), ptrs) }) } @@ -388,9 +514,9 @@ impl Store { } else { let path_ptr = self.intern_symbol_path(sym.path()); let sym_ptr = if sym == &lurk_sym("nil") { - path_ptr.cast(Tag::Expr(Nil)) + Ptr::new(Tag::Expr(Nil), *path_ptr.pay()) } else if sym.is_keyword() { - path_ptr.cast(Tag::Expr(Key)) + Ptr::new(Tag::Expr(Key), *path_ptr.pay()) } else { path_ptr }; @@ -404,11 +530,13 @@ impl Store { fn fetch_symbol_path(&self, mut idx: usize) -> Option> { let mut path = vec![]; loop { - let (car, cdr) = self.fetch_2_ptrs(idx)?; - let string = self.fetch_string(car)?; + let [car_tag, car, cdr_tag, cdr] = self.fetch_raw_ptrs(idx)?; + assert_eq!(*car_tag, self.tag(Tag::Expr(Str))); + assert_eq!(*cdr_tag, self.tag(Tag::Expr(Sym))); + let string = self.fetch_string(&Ptr::new(Tag::Expr(Str), *car))?; path.push(string); match cdr { - Ptr::Atom(Tag::Expr(Sym), idx) => { + RawPtr::Atom(idx) => { if self.fetch_f(*idx)? == &F::ZERO { path.reverse(); return Some(path); @@ -416,7 +544,7 @@ impl Store { return None; } } - Ptr::Tuple2(Tag::Expr(Sym), idx_cdr) => idx = *idx_cdr, + RawPtr::Hash4(idx_cdr) => idx = *idx_cdr, _ => return None, } } @@ -426,8 +554,8 @@ impl Store { if let Some(sym) = self.ptr_symbol_cache.get(ptr) { Some(sym.clone()) } else { - match ptr { - Ptr::Atom(Tag::Expr(Sym), idx) => { + match (ptr.tag(), ptr.pay()) { + (Tag::Expr(Sym), RawPtr::Atom(idx)) => { if self.fetch_f(*idx)? == &F::ZERO { let sym = Symbol::root_sym(); self.ptr_symbol_cache.insert(*ptr, Box::new(sym.clone())); @@ -436,7 +564,7 @@ impl Store { None } } - Ptr::Atom(Tag::Expr(Key), idx) => { + (Tag::Expr(Key), RawPtr::Atom(idx)) => { if self.fetch_f(*idx)? == &F::ZERO { let key = Symbol::root_key(); self.ptr_symbol_cache.insert(*ptr, Box::new(key.clone())); @@ -445,13 +573,13 @@ impl Store { None } } - Ptr::Tuple2(Tag::Expr(Sym | Nil), idx) => { + (Tag::Expr(Sym | Nil), RawPtr::Hash4(idx)) => { let path = self.fetch_symbol_path(*idx)?; let sym = Symbol::sym_from_vec(path); self.ptr_symbol_cache.insert(*ptr, Box::new(sym.clone())); Some(sym) } - Ptr::Tuple2(Tag::Expr(Key), idx) => { + (Tag::Expr(Key), RawPtr::Hash4(idx)) => { let path = self.fetch_symbol_path(*idx)?; let key = Symbol::key_from_vec(path); self.ptr_symbol_cache.insert(*ptr, Box::new(key.clone())); @@ -478,15 +606,34 @@ impl Store { } } + #[inline] + pub fn intern_lurk_symbol(&self, name: &str) -> Ptr { + self.intern_symbol(&lurk_sym(name)) + } + + #[inline] + pub fn intern_nil(&self) -> Ptr { + self.intern_lurk_symbol("nil") + } + + #[inline] + pub fn intern_user_symbol(&self, name: &str) -> Ptr { + self.intern_symbol(&user_sym(name)) + } + + #[inline] + pub fn key(&self, name: &str) -> Ptr { + self.intern_symbol(&Symbol::key(&[name.to_string()])) + } + #[inline] pub fn add_comm(&self, hash: F, secret: F, payload: Ptr) { - self.comms - .insert(FWrap::(hash), Box::new((secret, payload))); + self.comms.insert(FWrap(hash), Box::new((secret, payload))); } #[inline] pub fn hide(&self, secret: F, payload: Ptr) -> Ptr { - self.comm(self.hide_and_return_z_payload(secret, payload).0) + self.comm(self.hide_ptr(secret, payload)) } pub fn hide_and_return_z_payload(&self, secret: F, payload: Ptr) -> (F, ZPtr) { @@ -508,49 +655,41 @@ impl Store { self.comms.get(&FWrap(hash)) } - #[inline] - pub fn intern_lurk_symbol(&self, name: &str) -> Ptr { - self.intern_symbol(&lurk_sym(name)) - } - - #[inline] - pub fn intern_nil(&self) -> Ptr { - self.intern_lurk_symbol("nil") - } - - #[inline] - pub fn intern_user_symbol(&self, name: &str) -> Ptr { - self.intern_symbol(&user_sym(name)) - } - - #[inline] - pub fn key(&self, name: &str) -> Ptr { - self.intern_symbol(&Symbol::key(&[name.to_string()])) + pub fn hide_ptr(&self, secret: F, payload: Ptr) -> F { + let hash = self.poseidon_cache.hash3(&[ + secret, + payload.tag().to_field(), + self.hash_raw_ptr(payload.pay()).0, + ]); + self.add_comm(hash, secret, payload); + hash } #[inline] pub fn cons(&self, car: Ptr, cdr: Ptr) -> Ptr { - self.intern_2_ptrs(Tag::Expr(Cons), car, cdr) + let ptrs = [car, cdr]; + self.intern_ptrs::<2>(Tag::Expr(Cons), ptrs) } #[inline] pub fn intern_fun(&self, arg: Ptr, body: Ptr, env: Ptr) -> Ptr { - self.intern_4_ptrs(Tag::Expr(Fun), arg, body, env, self.dummy()) + let ptrs = [arg, body, env, self.dummy()]; + self.intern_ptrs::<4>(Tag::Expr(Fun), ptrs) } #[inline] pub fn cont_outermost(&self) -> Ptr { - Ptr::Atom(Tag::Cont(Outermost), self.hash8zeros_idx) + Ptr::new(Tag::Cont(Outermost), RawPtr::Atom(self.hash8zeros_idx)) } #[inline] pub fn cont_error(&self) -> Ptr { - Ptr::Atom(Tag::Cont(ContTag::Error), self.hash8zeros_idx) + Ptr::new(Tag::Cont(ContTag::Error), RawPtr::Atom(self.hash8zeros_idx)) } #[inline] pub fn cont_terminal(&self) -> Ptr { - Ptr::Atom(Tag::Cont(Terminal), self.hash8zeros_idx) + Ptr::new(Tag::Cont(Terminal), RawPtr::Atom(self.hash8zeros_idx)) } pub fn car_cdr(&self, ptr: &Ptr) -> Result<(Ptr, Ptr)> { @@ -560,23 +699,32 @@ impl Store { Ok((nil, nil)) } Tag::Expr(Cons) => { - let Some(idx) = ptr.get_index2() else { + let Some(idx) = ptr.pay().get_hash4() else { bail!("malformed cons pointer") }; - match self.fetch_2_ptrs(idx) { - Some(res) => Ok(*res), + match self.fetch_raw_ptrs(idx) { + Some([car_tag, car, cdr_tag, cdr]) => { + let car_ptr = self.raw_to_ptr(car_tag, car).context("Not a pointer")?; + let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr).context("Not a pointer")?; + Ok((car_ptr, cdr_ptr)) + } None => bail!("car/cdr not found"), } } Tag::Expr(Str) => { - if self.is_zero(ptr) { - Ok((self.intern_nil(), self.zero(Tag::Expr(Str)))) + if self.is_zero(ptr.pay()) { + let nil_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); + Ok((self.intern_nil(), nil_str)) } else { - let Some(idx) = ptr.get_index2() else { + let Some(idx) = ptr.pay().get_hash4() else { bail!("malformed str pointer") }; - match self.fetch_2_ptrs(idx) { - Some(res) => Ok(*res), + match self.fetch_raw_ptrs(idx) { + Some([car_tag, car, cdr_tag, cdr]) => { + let car_ptr = self.raw_to_ptr(car_tag, car).context("Not a pointer")?; + let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr).context("Not a pointer")?; + Ok((car_ptr, cdr_ptr)) + } None => bail!("car/cdr not found"), } } @@ -591,7 +739,7 @@ impl Store { elts.into_iter() .rev() .fold(last.unwrap_or_else(|| self.intern_nil()), |acc, elt| { - self.intern_2_ptrs(Tag::Expr(Cons), elt, acc) + self.cons(elt, acc) }) } @@ -611,20 +759,25 @@ impl Store { /// Fetches a cons list that was interned. If the list is improper, the second /// element of the returned pair will carry the improper terminating value pub fn fetch_list(&self, ptr: &Ptr) -> Option<(Vec, Option)> { - match ptr { - Ptr::Tuple2(Tag::Expr(Nil), _) => Some((vec![], None)), - Ptr::Tuple2(Tag::Expr(Cons), mut idx) => { + if *ptr == self.intern_nil() { + return Some((vec![], None)); + } + match (ptr.tag(), ptr.pay()) { + (Tag::Expr(Nil), _) => panic!("Malformed nil expression"), + (Tag::Expr(Cons), RawPtr::Hash4(mut idx)) => { let mut list = vec![]; let mut last = None; - while let Some((car, cdr)) = self.fetch_2_ptrs(idx) { - list.push(*car); - match cdr.tag() { + while let Some([car_tag, car, cdr_tag, cdr]) = self.fetch_raw_ptrs(idx) { + let car_ptr = self.raw_to_ptr(car_tag, car)?; + let cdr_ptr = self.raw_to_ptr(cdr_tag, cdr)?; + list.push(car_ptr); + match cdr_ptr.tag() { Tag::Expr(Nil) => break, Tag::Expr(Cons) => { - idx = cdr.get_index2()?; + idx = cdr.get_hash4()?; } _ => { - last = Some(*cdr); + last = Some(cdr_ptr); break; } } @@ -693,26 +846,6 @@ impl Store { self.read(State::init_lurk_state().rccell(), input) } - #[inline] - pub fn expect_f(&self, idx: usize) -> &F { - self.fetch_f(idx).expect("Index missing from f_elts") - } - - #[inline] - pub fn expect_2_ptrs(&self, idx: usize) -> &(Ptr, Ptr) { - self.fetch_2_ptrs(idx).expect("Index missing from tuple2") - } - - #[inline] - pub fn expect_3_ptrs(&self, idx: usize) -> &(Ptr, Ptr, Ptr) { - self.fetch_3_ptrs(idx).expect("Index missing from tuple3") - } - - #[inline] - pub fn expect_4_ptrs(&self, idx: usize) -> &(Ptr, Ptr, Ptr, Ptr) { - self.fetch_4_ptrs(idx).expect("Index missing from tuple4") - } - /// Recursively hashes the children of a `Ptr` in order to obtain its /// corresponding `ZPtr`. While traversing a `Ptr` tree, it consults the /// cache of `Ptr`s that have already been hydrated and also populates this @@ -721,88 +854,67 @@ impl Store { /// Warning: without cache hits, this function might blow up Rust's recursion /// depth limit. This limitation is circumvented by calling `hydrate_z_cache` /// beforehand or by using `hash_ptr` instead, which is slightly slower. - fn hash_ptr_unsafe(&self, ptr: &Ptr) -> ZPtr { + fn hash_raw_ptr_unsafe(&self, ptr: &RawPtr) -> FWrap { match ptr { - Ptr::Atom(tag, idx) => { - if let Some(z_ptr) = self.z_cache.get(ptr) { - *z_ptr + RawPtr::Atom(idx) => FWrap(*self.expect_f(*idx)), + RawPtr::Hash3(idx) => { + if let Some(z) = self.z_cache.get(ptr) { + *z } else { - let z_ptr = ZPtr::from_parts(*tag, *self.expect_f(*idx)); - self.z_cache.insert(*ptr, Box::new(z_ptr)); - self.inverse_z_cache.insert(z_ptr, Box::new(*ptr)); - z_ptr + let children_ptrs = self.expect_raw_ptrs::<3>(*idx); + let mut children_zs = [F::ZERO; 3]; + for (idx, child_ptr) in children_ptrs.iter().enumerate() { + children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; + } + let z = FWrap(self.poseidon_cache.hash3(&children_zs)); + self.z_cache.insert(*ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(*ptr)); + z } } - Ptr::Tuple2(tag, idx) => { - if let Some(z_ptr) = self.z_cache.get(ptr) { - *z_ptr + RawPtr::Hash4(idx) => { + if let Some(z) = self.z_cache.get(ptr) { + *z } else { - let (a, b) = self.expect_2_ptrs(*idx); - let a = self.hash_ptr_unsafe(a); - let b = self.hash_ptr_unsafe(b); - let z_ptr = ZPtr::from_parts( - *tag, - self.poseidon_cache.hash4(&[ - a.tag_field(), - *a.value(), - b.tag_field(), - *b.value(), - ]), - ); - self.z_cache.insert(*ptr, Box::new(z_ptr)); - self.inverse_z_cache.insert(z_ptr, Box::new(*ptr)); - z_ptr + let children_ptrs = self.expect_raw_ptrs::<4>(*idx); + let mut children_zs = [F::ZERO; 4]; + for (idx, child_ptr) in children_ptrs.iter().enumerate() { + children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; + } + let z = FWrap(self.poseidon_cache.hash4(&children_zs)); + self.z_cache.insert(*ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(*ptr)); + z } } - Ptr::Tuple3(tag, idx) => { - if let Some(z_ptr) = self.z_cache.get(ptr) { - *z_ptr + RawPtr::Hash6(idx) => { + if let Some(z) = self.z_cache.get(ptr) { + *z } else { - let (a, b, c) = self.expect_3_ptrs(*idx); - let a = self.hash_ptr_unsafe(a); - let b = self.hash_ptr_unsafe(b); - let c = self.hash_ptr_unsafe(c); - let z_ptr = ZPtr::from_parts( - *tag, - self.poseidon_cache.hash6(&[ - a.tag_field(), - *a.value(), - b.tag_field(), - *b.value(), - c.tag_field(), - *c.value(), - ]), - ); - self.z_cache.insert(*ptr, Box::new(z_ptr)); - self.inverse_z_cache.insert(z_ptr, Box::new(*ptr)); - z_ptr + let children_ptrs = self.expect_raw_ptrs::<6>(*idx); + let mut children_zs = [F::ZERO; 6]; + for (idx, child_ptr) in children_ptrs.iter().enumerate() { + children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; + } + let z = FWrap(self.poseidon_cache.hash6(&children_zs)); + self.z_cache.insert(*ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(*ptr)); + z } } - Ptr::Tuple4(tag, idx) => { - if let Some(z_ptr) = self.z_cache.get(ptr) { - *z_ptr + RawPtr::Hash8(idx) => { + if let Some(z) = self.z_cache.get(ptr) { + *z } else { - let (a, b, c, d) = self.expect_4_ptrs(*idx); - let a = self.hash_ptr_unsafe(a); - let b = self.hash_ptr_unsafe(b); - let c = self.hash_ptr_unsafe(c); - let d = self.hash_ptr_unsafe(d); - let z_ptr = ZPtr::from_parts( - *tag, - self.poseidon_cache.hash8(&[ - a.tag_field(), - *a.value(), - b.tag_field(), - *b.value(), - c.tag_field(), - *c.value(), - d.tag_field(), - *d.value(), - ]), - ); - self.z_cache.insert(*ptr, Box::new(z_ptr)); - self.inverse_z_cache.insert(z_ptr, Box::new(*ptr)); - z_ptr + let children_ptrs = self.expect_raw_ptrs::<8>(*idx); + let mut children_zs = [F::ZERO; 8]; + for (idx, child_ptr) in children_ptrs.iter().enumerate() { + children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; + } + let z = FWrap(self.poseidon_cache.hash8(&children_zs)); + self.z_cache.insert(*ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(*ptr)); + z } } } @@ -814,10 +926,10 @@ impl Store { /// limit in `hash_ptr_unsafe`. So we move in smaller chunks from left to /// right, populating the `z_cache`, which can rescue `hash_ptr_unsafe` from /// dangerously deep recursions - fn hydrate_z_cache_with_ptrs(&self, ptrs: &[&Ptr]) { + fn hydrate_z_cache_with_ptrs(&self, ptrs: &[&RawPtr]) { ptrs.chunks(256).for_each(|chunk| { chunk.par_iter().for_each(|ptr| { - self.hash_ptr_unsafe(ptr); + self.hash_raw_ptr_unsafe(ptr); }); }); } @@ -845,17 +957,17 @@ impl Store { /// Safe version of `hash_ptr_unsafe` that doesn't hit a stack overflow by /// precomputing the pointers that need to be hashed in order to hash the /// provided `ptr` - pub fn hash_ptr(&self, ptr: &Ptr) -> ZPtr { + pub fn hash_raw_ptr(&self, ptr: &RawPtr) -> FWrap { if self.is_below_safe_threshold() { // just run `hash_ptr_unsafe` for extra speed when the dehydrated // queue is small enough - return self.hash_ptr_unsafe(ptr); + return self.hash_raw_ptr_unsafe(ptr); } - let mut ptrs: IndexSet<&Ptr> = IndexSet::default(); + let mut ptrs: IndexSet<&RawPtr> = IndexSet::default(); let mut stack = vec![ptr]; macro_rules! feed_loop { ($x:expr) => { - if $x.is_tuple() { + if $x.is_hash() { if self.z_cache.get($x).is_none() { if ptrs.insert($x) { stack.push($x); @@ -866,22 +978,28 @@ impl Store { } while let Some(ptr) = stack.pop() { match ptr { - Ptr::Atom(..) => (), - Ptr::Tuple2(_, idx) => { - let (a, b) = self.expect_2_ptrs(*idx); - for ptr in [a, b] { + RawPtr::Atom(..) => (), + RawPtr::Hash3(idx) => { + let ptrs = self.expect_raw_ptrs::<3>(*idx); + for ptr in ptrs { + feed_loop!(ptr) + } + } + RawPtr::Hash4(idx) => { + let ptrs = self.expect_raw_ptrs::<4>(*idx); + for ptr in ptrs { feed_loop!(ptr) } } - Ptr::Tuple3(_, idx) => { - let (a, b, c) = self.expect_3_ptrs(*idx); - for ptr in [a, b, c] { + RawPtr::Hash6(idx) => { + let ptrs = self.expect_raw_ptrs::<6>(*idx); + for ptr in ptrs { feed_loop!(ptr) } } - Ptr::Tuple4(_, idx) => { - let (a, b, c, d) = self.expect_4_ptrs(*idx); - for ptr in [a, b, c, d] { + RawPtr::Hash8(idx) => { + let ptrs = self.expect_raw_ptrs::<8>(*idx); + for ptr in ptrs { feed_loop!(ptr) } } @@ -890,7 +1008,11 @@ impl Store { ptrs.reverse(); self.hydrate_z_cache_with_ptrs(&ptrs.into_iter().collect::>()); // Now it's okay to call `hash_ptr_unsafe` - self.hash_ptr_unsafe(ptr) + self.hash_raw_ptr_unsafe(ptr) + } + + pub fn hash_ptr(&self, ptr: &Ptr) -> ZPtr { + ZPtr::from_parts(*ptr.tag(), self.hash_raw_ptr(ptr.pay()).0) } /// Constructs a vector of scalars that correspond to tags and hashes computed @@ -898,14 +1020,24 @@ impl Store { pub fn to_scalar_vector(&self, ptrs: &[Ptr]) -> Vec { ptrs.iter() .fold(Vec::with_capacity(2 * ptrs.len()), |mut acc, ptr| { - let z_ptr = self.hash_ptr(ptr); - acc.push(z_ptr.tag_field()); - acc.push(*z_ptr.value()); + let tag = ptr.tag().to_field(); + let pay = self.hash_raw_ptr(ptr.pay()).0; + acc.push(tag); + acc.push(pay); acc }) } + pub fn to_scalar_vector_raw(&self, ptrs: &[RawPtr]) -> Vec { + ptrs.iter().map(|ptr| self.hash_raw_ptr(ptr).0).collect() + } + /// Equality of the content-addressed versions of two pointers + #[inline] + pub fn raw_ptr_eq(&self, a: &RawPtr, b: &RawPtr) -> bool { + self.hash_raw_ptr(a) == self.hash_raw_ptr(b) + } + #[inline] pub fn ptr_eq(&self, a: &Ptr, b: &Ptr) -> bool { self.hash_ptr(a) == self.hash_ptr(b) @@ -915,11 +1047,16 @@ impl Store { /// `inverse_z_cache`. If the mapping is not there, returns an atom pointer /// with the same tag and value #[inline] - pub fn to_ptr(&self, z_ptr: &ZPtr) -> Ptr { + pub fn to_raw_ptr(&self, z: &FWrap) -> RawPtr { self.inverse_z_cache - .get(z_ptr) + .get(z) .cloned() - .unwrap_or_else(|| self.opaque(*z_ptr)) + .unwrap_or_else(|| self.opaque(*z)) + } + + #[inline] + pub fn to_ptr(&self, z_ptr: &ZPtr) -> Ptr { + Ptr::new(*z_ptr.tag(), self.to_raw_ptr(&FWrap(*z_ptr.value()))) } } @@ -957,6 +1094,7 @@ impl Ptr { } Char => { if let Some(c) = self + .pay() .get_atom() .map(|idx| store.expect_f(idx)) .and_then(F::to_char) @@ -986,7 +1124,7 @@ impl Ptr { } } Num => { - if let Some(f) = self.get_atom().map(|idx| store.expect_f(idx)) { + if let Some(f) = self.pay().get_atom().map(|idx| store.expect_f(idx)) { if let Some(u) = f.to_u64() { u.to_string() } else { @@ -998,6 +1136,7 @@ impl Ptr { } U64 => { if let Some(u) = self + .pay() .get_atom() .map(|idx| store.expect_f(idx)) .and_then(F::to_u64) @@ -1007,10 +1146,10 @@ impl Ptr { "".into() } } - Fun => match self.get_index4() { + Fun => match self.pay().get_hash8() { None => "".into(), Some(idx) => { - if let Some((vars, body, ..)) = store.fetch_4_ptrs(idx) { + if let Some([vars, body, _, _]) = store.fetch_ptrs::<8, 4>(idx) { match vars.tag() { Tag::Expr(Nil) => { format!("", body.fmt_to_string(store, state)) @@ -1029,10 +1168,10 @@ impl Ptr { } } }, - Thunk => match self.get_index2() { + Thunk => match self.pay().get_hash4() { None => "".into(), Some(idx) => { - if let Some((val, cont)) = store.fetch_2_ptrs(idx) { + if let Some([val, cont]) = store.fetch_ptrs::<4, 2>(idx) { format!( "Thunk{{ value: {} => cont: {} }}", val.fmt_to_string(store, state), @@ -1043,7 +1182,7 @@ impl Ptr { } } }, - Comm => match self.get_atom() { + Comm => match self.pay().get_atom() { Some(idx) => { let f = store.expect_f(idx); if store.comms.get(&FWrap(*f)).is_some() { @@ -1054,10 +1193,10 @@ impl Ptr { } None => "".into(), }, - Cproc => match self.get_index2() { + Cproc => match self.pay().get_hash4() { None => "".into(), Some(idx) => { - if let Some((cproc_name, args)) = store.fetch_2_ptrs(idx) { + if let Some([cproc_name, args]) = store.fetch_ptrs::<4, 2>(idx) { format!( "", cproc_name.fmt_to_string(store, state), @@ -1116,10 +1255,10 @@ impl Ptr { store: &Store, state: &State, ) -> String { - match self.get_index4() { + match self.pay().get_hash8() { None => format!(""), Some(idx) => { - if let Some((a, cont, ..)) = store.fetch_4_ptrs(idx) { + if let Some([a, cont, _, _]) = store.fetch_ptrs::<8, 4>(idx) { format!( "{name}{{ {field}: {}, continuation: {} }}", a.fmt_to_string(store, state), @@ -1139,10 +1278,10 @@ impl Ptr { store: &Store, state: &State, ) -> String { - match self.get_index4() { + match self.pay().get_hash8() { None => format!(""), Some(idx) => { - if let Some((a, b, cont, _)) = store.fetch_4_ptrs(idx) { + if let Some([a, b, cont, _]) = store.fetch_ptrs::<8, 4>(idx) { let (fa, fb) = fields; format!( "{name}{{ {fa}: {}, {fb}: {}, continuation: {} }}", @@ -1164,10 +1303,10 @@ impl Ptr { store: &Store, state: &State, ) -> String { - match self.get_index4() { + match self.pay().get_hash8() { None => format!(""), Some(idx) => { - if let Some((a, b, c, cont)) = store.fetch_4_ptrs(idx) { + if let Some([a, b, c, cont]) = store.fetch_ptrs::<8, 4>(idx) { let (fa, fb, fc) = fields; format!( "{name}{{ {fa}: {}, {fb}: {}, {fc}: {}, continuation: {} }}", @@ -1183,262 +1322,3 @@ impl Ptr { } } } - -#[cfg(test)] -mod tests { - use ff::Field; - use pasta_curves::pallas::Scalar as Fr; - use proptest::prelude::*; - - use crate::{ - field::LurkField, - lem::Tag, - parser::position::Pos, - state::{initial_lurk_state, lurk_sym}, - syntax::Syntax, - tag::{ExprTag, Tag as TagTrait}, - Num, Symbol, - }; - - use super::{Ptr, Store}; - - #[test] - fn test_car_cdr() { - let store = Store::::default(); - - // empty list - let nil = store.intern_nil(); - let (car, cdr) = store.car_cdr(&nil).unwrap(); - assert_eq!((&car, &cdr), (&nil, &nil)); - - // regular cons - let one = store.num_u64(1); - let a = store.char('a'); - let one_a = store.cons(one, a); - let (car, cdr) = store.car_cdr(&one_a).unwrap(); - assert_eq!((&one, &a), (&car, &cdr)); - - // string - let abc = store.intern_string("abc"); - let bc = store.intern_string("bc"); - let (car, cdr) = store.car_cdr(&abc).unwrap(); - assert_eq!((&a, &bc), (&car, &cdr)); - - // empty string - let empty_str = store.intern_string(""); - let (car, cdr) = store.car_cdr(&empty_str).unwrap(); - assert_eq!((&nil, &empty_str), (&car, &cdr)); - } - - #[test] - fn test_list() { - let store = Store::::default(); - let state = initial_lurk_state(); - - // empty list - let list = store.list(vec![]); - let nil = store.intern_nil(); - assert_eq!(&list, &nil); - let (elts, non_nil) = store.fetch_list(&list).unwrap(); - assert!(elts.is_empty()); - assert!(non_nil.is_none()); - - // proper list - let a = store.char('a'); - let b = store.char('b'); - let list = store.list(vec![a, b]); - assert_eq!(list.fmt_to_string(&store, state), "('a' 'b')"); - let (elts, non_nil) = store.fetch_list(&list).unwrap(); - assert_eq!(elts.len(), 2); - assert_eq!((&elts[0], &elts[1]), (&a, &b)); - assert!(non_nil.is_none()); - - // improper list - let c = store.char('c'); - let b_c = store.cons(b, c); - let a_b_c = store.cons(a, b_c); - let a_b_c_ = store.improper_list(vec![a, b], c); - assert_eq!(a_b_c, a_b_c_); - assert_eq!(a_b_c.fmt_to_string(&store, state), "('a' 'b' . 'c')"); - let (elts, non_nil) = store.fetch_list(&a_b_c).unwrap(); - assert_eq!(elts.len(), 2); - assert_eq!((&elts[0], &elts[1]), (&a, &b)); - assert_eq!(non_nil, Some(c)); - } - - #[test] - fn test_basic_hashing() { - let store = Store::::default(); - let zero = Fr::zero(); - let zero_tag = Tag::try_from(0).unwrap(); - let foo = store.intern_atom(zero_tag, zero); - - let z_foo = store.hash_ptr(&foo); - assert_eq!(z_foo.tag(), &zero_tag); - assert_eq!(z_foo.value(), &zero); - - let comm = store.hide(zero, foo); - assert_eq!(comm.tag(), &Tag::Expr(ExprTag::Comm)); - assert_eq!( - store.expect_f(comm.get_atom().unwrap()), - &store.poseidon_cache.hash3(&[zero; 3]) - ); - - let ptr2 = store.intern_2_ptrs(zero_tag, foo, foo); - let z_ptr2 = store.hash_ptr(&ptr2); - assert_eq!(z_ptr2.tag(), &zero_tag); - assert_eq!(z_ptr2.value(), &store.poseidon_cache.hash4(&[zero; 4])); - - let ptr3 = store.intern_3_ptrs(zero_tag, foo, foo, foo); - let z_ptr3 = store.hash_ptr(&ptr3); - assert_eq!(z_ptr3.tag(), &zero_tag); - assert_eq!(z_ptr3.value(), &store.poseidon_cache.hash6(&[zero; 6])); - - let ptr4 = store.intern_4_ptrs(zero_tag, foo, foo, foo, foo); - let z_ptr4 = store.hash_ptr(&ptr4); - assert_eq!(z_ptr4.tag(), &zero_tag); - assert_eq!(z_ptr4.value(), &store.poseidon_cache.hash8(&[zero; 8])); - } - - #[test] - fn test_display_opaque_knowledge() { - // bob creates a list - let store = Store::::default(); - let one = store.num_u64(1); - let two = store.num_u64(2); - let one_two = store.cons(one, two); - let hi = store.intern_string("hi"); - let z1 = store.hash_ptr(&hi); - let z2 = store.hash_ptr(&one_two); - let list = store.list(vec![one_two, hi]); - let z_list = store.hash_ptr(&list); - - // alice uses the hashed elements of the list to show that she - // can produce the same hash as the original z_list - let store = Store::::default(); - let a1 = store.opaque(z1); - let a2 = store.opaque(z2); - let list1 = store.list(vec![a1, a2]); - let list2 = store.list(vec![a2, a1]); - let z_list1 = store.hash_ptr(&list1); - let z_list2 = store.hash_ptr(&list2); - - // one of those lists should match the original - assert!(z_list == z_list1 || z_list == z_list2); - } - - #[test] - fn test_ptr_hashing_safety() { - let string = String::from_utf8(vec![b'0'; 4096]).unwrap(); - let store = Store::::default(); - let ptr = store.intern_string(&string); - // `hash_ptr_unsafe` would overflow the stack, whereas `hash_ptr` works - let x = store.hash_ptr(&ptr); - - let store = Store::::default(); - let ptr = store.intern_string(&string); - store.hydrate_z_cache(); - // but `hash_ptr_unsafe` works just fine after manual hydration - let y = store.hash_ptr_unsafe(&ptr); - - // and, of course, those functions result on the same `ZPtr` - assert_eq!(x, y); - } - - #[test] - fn string_hashing() { - let s = &Store::::default(); - let hi_ptr = s.intern_string("hi"); - - let hi_hash_manual = s.poseidon_cache.hash4(&[ - ExprTag::Char.to_field(), - Fr::from_char('h'), - ExprTag::Str.to_field(), - s.poseidon_cache.hash4(&[ - ExprTag::Char.to_field(), - Fr::from_char('i'), - ExprTag::Str.to_field(), - Fr::ZERO, - ]), - ]); - - let hi_hash = s.hash_ptr(&hi_ptr).1; - assert_eq!(hi_hash, hi_hash_manual); - } - - #[test] - fn symbol_hashing() { - let s = &Store::::default(); - let foo_ptr = s.intern_string("foo"); - let bar_ptr = s.intern_string("bar"); - let foo_bar_ptr = s.intern_symbol(&Symbol::sym_from_vec(vec!["foo".into(), "bar".into()])); - - let foo_z_ptr = s.hash_ptr(&foo_ptr); - let bar_z_ptr = s.hash_ptr(&bar_ptr); - - let foo_bar_hash_manual = s.poseidon_cache.hash4(&[ - ExprTag::Str.to_field(), - bar_z_ptr.1, - ExprTag::Sym.to_field(), - s.poseidon_cache.hash4(&[ - ExprTag::Str.to_field(), - foo_z_ptr.1, - ExprTag::Sym.to_field(), - Fr::ZERO, - ]), - ]); - - let foo_bar_hash = s.hash_ptr(&foo_bar_ptr).1; - assert_eq!(foo_bar_hash, foo_bar_hash_manual); - } - - // helper function to test syntax interning roundtrip - fn fetch_syntax(ptr: Ptr, store: &Store) -> Syntax { - match ptr { - Ptr::Atom(Tag::Expr(ExprTag::Num), idx) => { - Syntax::Num(Pos::No, Num::Scalar(*store.expect_f(idx))) - } - Ptr::Atom(Tag::Expr(ExprTag::Char), idx) => { - Syntax::Char(Pos::No, store.expect_f(idx).to_char().unwrap()) - } - Ptr::Atom(Tag::Expr(ExprTag::U64), idx) => Syntax::UInt( - Pos::No, - crate::UInt::U64(store.expect_f(idx).to_u64_unchecked()), - ), - Ptr::Atom(Tag::Expr(ExprTag::Sym | ExprTag::Key), _) - | Ptr::Tuple2(Tag::Expr(ExprTag::Sym | ExprTag::Key), _) => { - Syntax::Symbol(Pos::No, store.fetch_symbol(&ptr).unwrap().into()) - } - Ptr::Atom(Tag::Expr(ExprTag::Str), _) | Ptr::Tuple2(Tag::Expr(ExprTag::Str), _) => { - Syntax::String(Pos::No, store.fetch_string(&ptr).unwrap()) - } - Ptr::Tuple2(Tag::Expr(ExprTag::Cons), _) => { - let (elts, last) = store.fetch_list(&ptr).unwrap(); - let elts = elts - .into_iter() - .map(|e| fetch_syntax(e, store)) - .collect::>(); - if let Some(last) = last { - Syntax::Improper(Pos::No, elts, fetch_syntax(last, store).into()) - } else { - Syntax::List(Pos::No, elts) - } - } - Ptr::Tuple2(Tag::Expr(ExprTag::Nil), _) => { - Syntax::Symbol(Pos::No, lurk_sym("nil").into()) - } - _ => unreachable!(), - } - } - - proptest! { - #[test] - fn syntax_roundtrip(x in any::>()) { - let store = Store::::default(); - let ptr1 = store.intern_syntax(x); - let y = fetch_syntax(ptr1, &store); - let ptr2 = store.intern_syntax(y); - assert_eq!(ptr1, ptr2); - } - } -} diff --git a/src/lem/tests/nivc_steps.rs b/src/lem/tests/nivc_steps.rs index 64b3e11ba4..fe741e0d57 100644 --- a/src/lem/tests/nivc_steps.rs +++ b/src/lem/tests/nivc_steps.rs @@ -78,12 +78,12 @@ fn test_nivc_steps() { let expr = cproc_input.pop().unwrap(); let idx = expr.get_index2().unwrap(); - let (_, args) = store.expect_2_ptrs(idx); + let [_, args] = store.expect_2_ptrs(idx); let new_name = user_sym("cproc-dumb-not"); let new_expr = store.intern_2_ptrs( Tag::Expr(ExprTag::Cproc), store.intern_symbol(&new_name), - *args, + args, ); // `cproc` can't reduce the altered cproc input (with the wrong name) From fe8e60255b62a642fbe787577466bc8f83b8189d Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Sun, 31 Dec 2023 21:24:16 -0300 Subject: [PATCH 05/12] Store simplification --- src/cli/repl/meta_cmd.rs | 2 +- src/cli/zstore.rs | 4 +- src/lem/interpreter.rs | 2 +- src/lem/store.rs | 165 +++++++++++++++++++-------------------- 4 files changed, 82 insertions(+), 91 deletions(-) diff --git a/src/cli/repl/meta_cmd.rs b/src/cli/repl/meta_cmd.rs index 5a016d59be..53c75afc57 100644 --- a/src/cli/repl/meta_cmd.rs +++ b/src/cli/repl/meta_cmd.rs @@ -622,7 +622,7 @@ impl MetaCmd { .store .open(hash) .expect("data must have been committed"); - repl.hide(*secret, *fun) + repl.hide(secret, fun) }, }; } diff --git a/src/cli/zstore.rs b/src/cli/zstore.rs index 65d148639c..a4379126ad 100644 --- a/src/cli/zstore.rs +++ b/src/cli/zstore.rs @@ -130,9 +130,7 @@ impl ZDag { } else { let ptr = match self.get_type(z_ptr) { None => bail!("Couldn't find ZPtr on ZStore"), - Some(ZPtrType::Atom) => { - store.intern_atom_hydrated(*z_ptr.tag(), *z_ptr.value(), *z_ptr) - } + Some(ZPtrType::Atom) => store.intern_atom(*z_ptr.tag(), *z_ptr.value()), Some(ZPtrType::Tuple2(z1, z2)) => { let ptr1 = self.populate_store(z1, store, cache)?; let ptr2 = self.populate_store(z2, store, cache)?; diff --git a/src/lem/interpreter.rs b/src/lem/interpreter.rs index 6fb04aefac..87372882ee 100644 --- a/src/lem/interpreter.rs +++ b/src/lem/interpreter.rs @@ -427,7 +427,7 @@ impl Block { bail!("{comm} is not a comm pointer") }; let hash = *store.expect_f(*hash); - let Some((secret, ptr)) = store.open(hash).cloned() else { + let Some((secret, ptr)) = store.open(hash) else { bail!("No committed data for hash {}", &hash.hex_digits()) }; bindings.insert_ptr(tgt_ptr.clone(), ptr); diff --git a/src/lem/store.rs b/src/lem/store.rs index ba38240235..6b4250d928 100644 --- a/src/lem/store.rs +++ b/src/lem/store.rs @@ -43,8 +43,6 @@ pub struct Store { ptr_string_cache: FrozenMap, ptr_symbol_cache: FrozenMap>, - comms: FrozenMap, Box<(F, Ptr)>>, // hash -> (secret, src) - pub poseidon_cache: PoseidonCache, pub inverse_poseidon_cache: InversePoseidonCache, @@ -83,7 +81,6 @@ impl Default for Store { symbol_ptr_cache: Default::default(), ptr_string_cache: Default::default(), ptr_symbol_cache: Default::default(), - comms: Default::default(), poseidon_cache, inverse_poseidon_cache: Default::default(), dehydrated: Default::default(), @@ -146,6 +143,35 @@ impl Store { self.expect_f(self.hash8zeros_idx) } + // Since the `generic_const_exprs` feature is still unstable, we cannot substitute `N * 2` + // for generic const `P` and remove it completely, so we must keep it and do a dynamic assertion + // that it equals `N * 2`. This is not very ergonomic though, since we must add turbofishes + // like `::<6, 3>` instead of the simpler `::<3>`. Could we maybe create a macro for these functions? + #[inline] + pub fn ptrs_to_raw_ptrs(&self, ptrs: &[Ptr; P]) -> [RawPtr; N] { + assert_eq!(P * 2, N); + let mut raw_ptrs = [self.raw_zero(); N]; + for i in 0..P { + raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); + raw_ptrs[2 * i + 1] = *ptrs[i].pay(); + } + raw_ptrs + } + + #[inline] + pub fn raw_ptrs_to_ptrs( + &self, + raw_ptrs: &[RawPtr; N], + ) -> Option<[Ptr; P]> { + assert_eq!(P * 2, N); + let mut ptrs = [self.dummy(); P]; + for i in 0..P { + let tag = self.fetch_tag(&raw_ptrs[2 * i])?; + ptrs[i] = Ptr::new(tag, raw_ptrs[2 * i + 1]) + } + Some(ptrs) + } + #[inline] pub fn intern_f(&self, f: F) -> (usize, bool) { self.f_elts.insert_probe(Box::new(FWrap(f))) @@ -164,20 +190,7 @@ impl Store { /// Creates a `RawPtr` that's a parent of `N` children pub fn intern_raw_ptrs(&self, ptrs: [RawPtr; N]) -> RawPtr { - macro_rules! intern { - ($Hash:ident, $hash:ident, $n:expr) => {{ - let ptrs = unsafe { std::mem::transmute::<&[RawPtr; N], &[RawPtr; $n]>(&ptrs) }; - let (idx, inserted) = self.$hash.insert_probe(Box::new(*ptrs)); - (RawPtr::$Hash(idx), inserted) - }}; - } - let (ptr, inserted) = match N { - 3 => intern!(Hash3, hash3, 3), - 4 => intern!(Hash4, hash4, 4), - 6 => intern!(Hash6, hash6, 6), - 8 => intern!(Hash8, hash8, 8), - _ => unimplemented!(), - }; + let (ptr, inserted) = self.intern_raw_ptrs_internal::(ptrs); if inserted { // this is for `hydrate_z_cache` self.dehydrated.load().push(Box::new(ptr)); @@ -193,71 +206,48 @@ impl Store { ptrs: [RawPtr; N], z: FWrap, ) -> RawPtr { + let (ptr, _) = self.intern_raw_ptrs_internal::(ptrs); + self.z_cache.insert(ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(ptr)); + ptr + } + + #[inline] + fn intern_raw_ptrs_internal(&self, ptrs: [RawPtr; N]) -> (RawPtr, bool) { macro_rules! intern { ($Hash:ident, $hash:ident, $n:expr) => {{ let ptrs = unsafe { std::mem::transmute::<&[RawPtr; N], &[RawPtr; $n]>(&ptrs) }; - let (idx, _) = self.$hash.insert_probe(Box::new(*ptrs)); - RawPtr::$Hash(idx) + let (idx, inserted) = self.$hash.insert_probe(Box::new(*ptrs)); + (RawPtr::$Hash(idx), inserted) }}; } - let ptr = match N { + match N { 3 => intern!(Hash3, hash3, 3), 4 => intern!(Hash4, hash4, 4), 6 => intern!(Hash6, hash6, 6), 8 => intern!(Hash8, hash8, 8), _ => unimplemented!(), - }; - self.z_cache.insert(ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(ptr)); - ptr + } } /// Creates a `Ptr` that's a parent of `N` children - pub fn intern_ptrs(&self, tag: Tag, ptrs: [Ptr; N]) -> Ptr { - macro_rules! intern { - ($n:expr) => {{ - let mut raw_ptrs = [self.raw_zero(); $n * 2]; - for i in 0..$n { - raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); - raw_ptrs[2 * i + 1] = *ptrs[i].pay(); - } - self.intern_raw_ptrs::<{ $n * 2 }>(raw_ptrs) - }}; - } - let pay = match N { - 2 => intern!(2), - 3 => intern!(3), - 4 => intern!(4), - _ => unimplemented!(), - }; + pub fn intern_ptrs(&self, tag: Tag, ptrs: [Ptr; P]) -> Ptr { + let raw_ptrs = self.ptrs_to_raw_ptrs::(&ptrs); + let pay = self.intern_raw_ptrs::(raw_ptrs); Ptr::new(tag, pay) } /// Similar to `intern_ptrs` but doesn't add the resulting pointer to /// `dehydrated`. This function is used when converting a `ZStore` to a /// `Store`. - pub fn intern_ptrs_hydrated( + pub fn intern_ptrs_hydrated( &self, tag: Tag, - ptrs: [Ptr; N], + ptrs: [Ptr; P], z: FWrap, ) -> Ptr { - macro_rules! intern { - ($n:expr) => {{ - let mut raw_ptrs = [self.raw_zero(); $n * 2]; - for i in 0..$n { - raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); - raw_ptrs[2 * i + 1] = *ptrs[i].pay(); - } - self.intern_raw_ptrs_hydrated::<{ $n * 2 }>(raw_ptrs, z) - }}; - } - let pay = match N { - 2 => intern!(2), - 3 => intern!(3), - 4 => intern!(4), - _ => unimplemented!(), - }; + let raw_ptrs = self.ptrs_to_raw_ptrs::(&ptrs); + let pay = self.intern_raw_ptrs_hydrated::(raw_ptrs, z); Ptr::new(tag, pay) } @@ -287,15 +277,8 @@ impl Store { #[inline] pub fn fetch_ptrs(&self, idx: usize) -> Option<[Ptr; P]> { assert_eq!(P * 2, N); - let raw_ptrs = self - .fetch_raw_ptrs::(idx) - .expect("Index missing from store"); - let mut ptrs = [self.dummy(); P]; - for i in 0..P { - let tag = self.fetch_tag(&raw_ptrs[2 * i])?; - ptrs[i] = Ptr::new(tag, raw_ptrs[2 * i + 1]) - } - Some(ptrs) + let raw_ptrs = self.fetch_raw_ptrs::(idx)?; + self.raw_ptrs_to_ptrs(raw_ptrs) } #[inline] @@ -315,29 +298,26 @@ impl Store { .expect("Index missing from store") } - // TODO remove these functions - pub fn intern_atom_hydrated(&self, tag: Tag, f: F, _: ZPtr) -> Ptr { - Ptr::new(tag, self.intern_raw_atom(f)) - } + // TODO eventually deprecate these functions #[inline] pub fn intern_2_ptrs(&self, tag: Tag, a: Ptr, b: Ptr) -> Ptr { - self.intern_ptrs::<2>(tag, [a, b]) + self.intern_ptrs::<4, 2>(tag, [a, b]) } #[inline] pub fn intern_3_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr) -> Ptr { - self.intern_ptrs::<3>(tag, [a, b, c]) + self.intern_ptrs::<6, 3>(tag, [a, b, c]) } #[inline] pub fn intern_4_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, d: Ptr) -> Ptr { - self.intern_ptrs::<4>(tag, [a, b, c, d]) + self.intern_ptrs::<8, 4>(tag, [a, b, c, d]) } #[inline] pub fn intern_2_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, z: ZPtr) -> Ptr { - self.intern_ptrs_hydrated::<2>(tag, [a, b], FWrap(*z.value())) + self.intern_ptrs_hydrated::<4, 2>(tag, [a, b], FWrap(*z.value())) } #[inline] pub fn intern_3_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, z: ZPtr) -> Ptr { - self.intern_ptrs_hydrated::<3>(tag, [a, b, c], FWrap(*z.value())) + self.intern_ptrs_hydrated::<6, 3>(tag, [a, b, c], FWrap(*z.value())) } #[inline] pub fn intern_4_ptrs_hydrated( @@ -349,7 +329,7 @@ impl Store { d: Ptr, z: ZPtr, ) -> Ptr { - self.intern_ptrs_hydrated::<4>(tag, [a, b, c, d], FWrap(*z.value())) + self.intern_ptrs_hydrated::<8, 4>(tag, [a, b, c, d], FWrap(*z.value())) } #[inline] pub fn fetch_2_ptrs(&self, idx: usize) -> Option<[Ptr; 2]> { @@ -454,7 +434,7 @@ impl Store { let nil_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); let ptr = s.chars().rev().fold(nil_str, |acc, c| { let ptrs = [self.char(c), acc]; - self.intern_ptrs::<2>(Tag::Expr(Str), ptrs) + self.intern_ptrs::<4, 2>(Tag::Expr(Str), ptrs) }); self.string_ptr_cache.insert(s.to_string(), Box::new(ptr)); self.ptr_string_cache.insert(ptr, s.to_string()); @@ -504,7 +484,7 @@ impl Store { let zero_sym = Ptr::new(Tag::Expr(Sym), self.raw_zero()); path.iter().fold(zero_sym, |acc, s| { let ptrs = [self.intern_string(s), acc]; - self.intern_ptrs::<2>(Tag::Expr(Sym), ptrs) + self.intern_ptrs::<4, 2>(Tag::Expr(Sym), ptrs) }) } @@ -628,7 +608,16 @@ impl Store { #[inline] pub fn add_comm(&self, hash: F, secret: F, payload: Ptr) { - self.comms.insert(FWrap(hash), Box::new((secret, payload))); + let ptrs = [ + self.intern_raw_atom(secret), + self.tag(*payload.tag()), + *payload.pay(), + ]; + let (idx, _) = self.hash3.insert_probe(Box::new(ptrs)); + let ptr = RawPtr::Hash3(idx); + let z = FWrap(hash); + self.z_cache.insert(ptr, Box::new(z)); + self.inverse_z_cache.insert(z, Box::new(ptr)); } #[inline] @@ -651,8 +640,12 @@ impl Store { } #[inline] - pub fn open(&self, hash: F) -> Option<&(F, Ptr)> { - self.comms.get(&FWrap(hash)) + pub fn open(&self, hash: F) -> Option<(F, Ptr)> { + let cached = self.inverse_z_cache.get(&FWrap(hash))?; + let [f, tag, pay] = self.fetch_raw_ptrs::<3>(cached.get_hash3()?)?; + let f = self.fetch_f(f.get_atom()?)?; + let ptr = self.raw_to_ptr(tag, pay)?; + Some((*f, ptr)) } pub fn hide_ptr(&self, secret: F, payload: Ptr) -> F { @@ -668,13 +661,13 @@ impl Store { #[inline] pub fn cons(&self, car: Ptr, cdr: Ptr) -> Ptr { let ptrs = [car, cdr]; - self.intern_ptrs::<2>(Tag::Expr(Cons), ptrs) + self.intern_ptrs::<4, 2>(Tag::Expr(Cons), ptrs) } #[inline] pub fn intern_fun(&self, arg: Ptr, body: Ptr, env: Ptr) -> Ptr { let ptrs = [arg, body, env, self.dummy()]; - self.intern_ptrs::<4>(Tag::Expr(Fun), ptrs) + self.intern_ptrs::<8, 4>(Tag::Expr(Fun), ptrs) } #[inline] @@ -1185,7 +1178,7 @@ impl Ptr { Comm => match self.pay().get_atom() { Some(idx) => { let f = store.expect_f(idx); - if store.comms.get(&FWrap(*f)).is_some() { + if store.inverse_z_cache.get(&FWrap(*f)).is_some() { format!("(comm 0x{})", f.hex_digits()) } else { format!("", f.hex_digits()) From 7db1e0d0aa80096315693ee15adf44d3a3a29d86 Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Tue, 2 Jan 2024 19:16:12 -0300 Subject: [PATCH 06/12] Renames, comments, tests, simplifications --- src/cli/zstore.rs | 7 +- src/lem/interpreter.rs | 26 +-- src/lem/pointers.rs | 26 +-- src/lem/store.rs | 408 +++++++++++++++++++++++++++++++---------- 4 files changed, 346 insertions(+), 121 deletions(-) diff --git a/src/cli/zstore.rs b/src/cli/zstore.rs index a4379126ad..07351f4767 100644 --- a/src/cli/zstore.rs +++ b/src/cli/zstore.rs @@ -39,7 +39,7 @@ impl ZDag { *z_ptr } else { let tag = ptr.tag(); - let z_ptr = match ptr.pay() { + let z_ptr = match ptr.raw() { RawPtr::Atom(idx) => { let f = store.expect_f(*idx); let z_ptr = ZPtr::from_parts(*tag, *f); @@ -47,7 +47,10 @@ impl ZDag { z_ptr } RawPtr::Hash3(_) => { - todo!() + // `Hash3` is currently only used for commit caches, and in particular, they + // never appear as part of expressions, since commits are not pointers, but + // field elements + unreachable!() } RawPtr::Hash4(idx) => { let [a, b] = store.expect_2_ptrs(*idx); diff --git a/src/lem/interpreter.rs b/src/lem/interpreter.rs index 87372882ee..aa01851be4 100644 --- a/src/lem/interpreter.rs +++ b/src/lem/interpreter.rs @@ -229,8 +229,8 @@ impl Block { bindings.insert_bool(tgt.clone(), a || b); } Op::Add(tgt, a, b) => { - let a = *bindings.get_ptr(a)?.pay(); - let b = *bindings.get_ptr(b)?.pay(); + let a = *bindings.get_ptr(a)?.raw(); + let b = *bindings.get_ptr(b)?.raw(); let c = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let (f, g) = (store.expect_f(f), store.expect_f(g)); store.intern_atom(Tag::Expr(Num), *f + *g) @@ -240,8 +240,8 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::Sub(tgt, a, b) => { - let a = *bindings.get_ptr(a)?.pay(); - let b = *bindings.get_ptr(b)?.pay(); + let a = *bindings.get_ptr(a)?.raw(); + let b = *bindings.get_ptr(b)?.raw(); let c = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let (f, g) = (store.expect_f(f), store.expect_f(g)); store.intern_atom(Tag::Expr(Num), *f - *g) @@ -251,8 +251,8 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::Mul(tgt, a, b) => { - let a = *bindings.get_ptr(a)?.pay(); - let b = *bindings.get_ptr(b)?.pay(); + let a = *bindings.get_ptr(a)?.raw(); + let b = *bindings.get_ptr(b)?.raw(); let c = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let (f, g) = (store.expect_f(f), store.expect_f(g)); store.intern_atom(Tag::Expr(Num), *f * *g) @@ -262,8 +262,8 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::Div(tgt, a, b) => { - let a = *bindings.get_ptr(a)?.pay(); - let b = *bindings.get_ptr(b)?.pay(); + let a = *bindings.get_ptr(a)?.raw(); + let b = *bindings.get_ptr(b)?.raw(); let c = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let (f, g) = (store.expect_f(f), store.expect_f(g)); if g == &F::ZERO { @@ -276,8 +276,8 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::Lt(tgt, a, b) => { - let a = *bindings.get_ptr(a)?.pay(); - let b = *bindings.get_ptr(b)?.pay(); + let a = *bindings.get_ptr(a)?.raw(); + let b = *bindings.get_ptr(b)?.raw(); let c = if let (RawPtr::Atom(f_idx), RawPtr::Atom(g_idx)) = (a, b) { let f = *store.expect_f(f_idx); let g = *store.expect_f(g_idx); @@ -301,7 +301,7 @@ impl Block { } Op::Trunc(tgt, a, n) => { assert!(*n <= 64); - let a = *bindings.get_ptr(a)?.pay(); + let a = *bindings.get_ptr(a)?.raw(); let c = if let RawPtr::Atom(f_idx) = a { let f = *store.expect_f(f_idx); hints.bit_decomp.push(Some(SlotData { @@ -315,8 +315,8 @@ impl Block { bindings.insert_ptr(tgt.clone(), c); } Op::DivRem64(tgt, a, b) => { - let a = *bindings.get_ptr(a)?.pay(); - let b = *bindings.get_ptr(b)?.pay(); + let a = *bindings.get_ptr(a)?.raw(); + let b = *bindings.get_ptr(b)?.raw(); let (c1, c2) = if let (RawPtr::Atom(f), RawPtr::Atom(g)) = (a, b) { let f = *store.expect_f(f); let g = *store.expect_f(g); diff --git a/src/lem/pointers.rs b/src/lem/pointers.rs index bedd755506..e211b0c7da 100644 --- a/src/lem/pointers.rs +++ b/src/lem/pointers.rs @@ -80,13 +80,13 @@ impl RawPtr { #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] pub struct Ptr { tag: Tag, - pay: RawPtr, + raw: RawPtr, } impl Ptr { #[inline] - pub fn new(tag: Tag, pay: RawPtr) -> Self { - Ptr { tag, pay } + pub fn new(tag: Tag, raw: RawPtr) -> Self { + Ptr { tag, raw } } #[inline] @@ -95,14 +95,14 @@ impl Ptr { } #[inline] - pub fn pay(&self) -> &RawPtr { - &self.pay + pub fn raw(&self) -> &RawPtr { + &self.raw } #[inline] pub fn parts(&self) -> (&Tag, &RawPtr) { - let Ptr { tag, pay } = self; - (tag, pay) + let Ptr { tag, raw } = self; + (tag, raw) } #[inline] @@ -147,34 +147,34 @@ impl Ptr { #[inline] pub fn cast(self, tag: Tag) -> Self { - Ptr { tag, pay: self.pay } + Ptr { tag, raw: self.raw } } #[inline] pub fn get_atom(&self) -> Option { - self.pay().get_atom() + self.raw().get_atom() } #[inline] pub fn get_index2(&self) -> Option { - self.pay().get_hash4() + self.raw().get_hash4() } #[inline] pub fn get_index3(&self) -> Option { - self.pay().get_hash6() + self.raw().get_hash6() } #[inline] pub fn get_index4(&self) -> Option { - self.pay().get_hash8() + self.raw().get_hash8() } #[inline] pub fn atom(tag: Tag, idx: usize) -> Ptr { Ptr { tag, - pay: RawPtr::Atom(idx), + raw: RawPtr::Atom(idx), } } } diff --git a/src/lem/store.rs b/src/lem/store.rs index 6b4250d928..5a30c43d8a 100644 --- a/src/lem/store.rs +++ b/src/lem/store.rs @@ -143,21 +143,24 @@ impl Store { self.expect_f(self.hash8zeros_idx) } - // Since the `generic_const_exprs` feature is still unstable, we cannot substitute `N * 2` - // for generic const `P` and remove it completely, so we must keep it and do a dynamic assertion - // that it equals `N * 2`. This is not very ergonomic though, since we must add turbofishes - // like `::<6, 3>` instead of the simpler `::<3>`. Could we maybe create a macro for these functions? + /// Converts array of pointers of size `P` to array of raw pointers of size `N` such that `P = N * 2`. + /// Since the `generic_const_exprs` feature is still unstable, we cannot substitute `N * 2` + /// for generic const `P` and remove it completely, so we must keep it and do a dynamic assertion + /// that it equals `N * 2`. This is not very ergonomic though, since we must add turbofishes + /// like `::<6, 3>` instead of the simpler `::<3>`. Could we maybe create a macro for these functions? #[inline] pub fn ptrs_to_raw_ptrs(&self, ptrs: &[Ptr; P]) -> [RawPtr; N] { assert_eq!(P * 2, N); let mut raw_ptrs = [self.raw_zero(); N]; for i in 0..P { raw_ptrs[2 * i] = self.tag(*ptrs[i].tag()); - raw_ptrs[2 * i + 1] = *ptrs[i].pay(); + raw_ptrs[2 * i + 1] = *ptrs[i].raw(); } raw_ptrs } + /// Tries to convert array of raw pointers of size `N` to array of pointers of size `P = N * 2`. + /// It might fail since not all raw pointers represent valid tags. #[inline] pub fn raw_ptrs_to_ptrs( &self, @@ -233,8 +236,8 @@ impl Store { /// Creates a `Ptr` that's a parent of `N` children pub fn intern_ptrs(&self, tag: Tag, ptrs: [Ptr; P]) -> Ptr { let raw_ptrs = self.ptrs_to_raw_ptrs::(&ptrs); - let pay = self.intern_raw_ptrs::(raw_ptrs); - Ptr::new(tag, pay) + let payload = self.intern_raw_ptrs::(raw_ptrs); + Ptr::new(tag, payload) } /// Similar to `intern_ptrs` but doesn't add the resulting pointer to @@ -247,8 +250,8 @@ impl Store { z: FWrap, ) -> Ptr { let raw_ptrs = self.ptrs_to_raw_ptrs::(&ptrs); - let pay = self.intern_raw_ptrs_hydrated::(raw_ptrs, z); - Ptr::new(tag, pay) + let payload = self.intern_raw_ptrs_hydrated::(raw_ptrs, z); + Ptr::new(tag, payload) } #[inline] @@ -368,9 +371,9 @@ impl Store { TagTrait::from_field(f) } - pub fn raw_to_ptr(&self, tag: &RawPtr, pay: &RawPtr) -> Option { + pub fn raw_to_ptr(&self, tag: &RawPtr, raw: &RawPtr) -> Option { let tag = self.fetch_tag(tag)?; - Some(Ptr::new(tag, *pay)) + Some(Ptr::new(tag, *raw)) } #[inline] @@ -423,16 +426,16 @@ impl Store { /// Creates an atom pointer from a `ZPtr`, with its hash. Hashing /// such pointer will result on the same original `ZPtr` #[inline] - pub fn opaque(&self, z: FWrap) -> RawPtr { - self.intern_raw_atom(z.0) + pub fn opaque(&self, z: ZPtr) -> Ptr { + self.intern_atom(*z.tag(), *z.value()) } pub fn intern_string(&self, s: &str) -> Ptr { if let Some(ptr) = self.string_ptr_cache.get(s) { *ptr } else { - let nil_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); - let ptr = s.chars().rev().fold(nil_str, |acc, c| { + let empty_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); + let ptr = s.chars().rev().fold(empty_str, |acc, c| { let ptrs = [self.char(c), acc]; self.intern_ptrs::<4, 2>(Tag::Expr(Str), ptrs) }); @@ -452,7 +455,7 @@ impl Store { return None; } loop { - match *ptr.pay() { + match *ptr.raw() { RawPtr::Atom(idx) => { if self.fetch_f(idx)? == &F::ZERO { self.ptr_string_cache.insert(ptr, string.clone()); @@ -494,9 +497,9 @@ impl Store { } else { let path_ptr = self.intern_symbol_path(sym.path()); let sym_ptr = if sym == &lurk_sym("nil") { - Ptr::new(Tag::Expr(Nil), *path_ptr.pay()) + Ptr::new(Tag::Expr(Nil), *path_ptr.raw()) } else if sym.is_keyword() { - Ptr::new(Tag::Expr(Key), *path_ptr.pay()) + Ptr::new(Tag::Expr(Key), *path_ptr.raw()) } else { path_ptr }; @@ -534,7 +537,7 @@ impl Store { if let Some(sym) = self.ptr_symbol_cache.get(ptr) { Some(sym.clone()) } else { - match (ptr.tag(), ptr.pay()) { + match (ptr.tag(), ptr.raw()) { (Tag::Expr(Sym), RawPtr::Atom(idx)) => { if self.fetch_f(*idx)? == &F::ZERO { let sym = Symbol::root_sym(); @@ -611,7 +614,7 @@ impl Store { let ptrs = [ self.intern_raw_atom(secret), self.tag(*payload.tag()), - *payload.pay(), + *payload.raw(), ]; let (idx, _) = self.hash3.insert_probe(Box::new(ptrs)); let ptr = RawPtr::Hash3(idx); @@ -652,7 +655,7 @@ impl Store { let hash = self.poseidon_cache.hash3(&[ secret, payload.tag().to_field(), - self.hash_raw_ptr(payload.pay()).0, + self.hash_raw_ptr(payload.raw()).0, ]); self.add_comm(hash, secret, payload); hash @@ -692,7 +695,7 @@ impl Store { Ok((nil, nil)) } Tag::Expr(Cons) => { - let Some(idx) = ptr.pay().get_hash4() else { + let Some(idx) = ptr.raw().get_hash4() else { bail!("malformed cons pointer") }; match self.fetch_raw_ptrs(idx) { @@ -705,11 +708,11 @@ impl Store { } } Tag::Expr(Str) => { - if self.is_zero(ptr.pay()) { - let nil_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); - Ok((self.intern_nil(), nil_str)) + if self.is_zero(ptr.raw()) { + let empty_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); + Ok((self.intern_nil(), empty_str)) } else { - let Some(idx) = ptr.pay().get_hash4() else { + let Some(idx) = ptr.raw().get_hash4() else { bail!("malformed str pointer") }; match self.fetch_raw_ptrs(idx) { @@ -755,7 +758,7 @@ impl Store { if *ptr == self.intern_nil() { return Some((vec![], None)); } - match (ptr.tag(), ptr.pay()) { + match (ptr.tag(), ptr.raw()) { (Tag::Expr(Nil), _) => panic!("Malformed nil expression"), (Tag::Expr(Cons), RawPtr::Hash4(mut idx)) => { let mut list = vec![]; @@ -848,68 +851,29 @@ impl Store { /// depth limit. This limitation is circumvented by calling `hydrate_z_cache` /// beforehand or by using `hash_ptr` instead, which is slightly slower. fn hash_raw_ptr_unsafe(&self, ptr: &RawPtr) -> FWrap { - match ptr { - RawPtr::Atom(idx) => FWrap(*self.expect_f(*idx)), - RawPtr::Hash3(idx) => { - if let Some(z) = self.z_cache.get(ptr) { - *z - } else { - let children_ptrs = self.expect_raw_ptrs::<3>(*idx); - let mut children_zs = [F::ZERO; 3]; - for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; - } - let z = FWrap(self.poseidon_cache.hash3(&children_zs)); - self.z_cache.insert(*ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(*ptr)); - z - } - } - RawPtr::Hash4(idx) => { - if let Some(z) = self.z_cache.get(ptr) { - *z - } else { - let children_ptrs = self.expect_raw_ptrs::<4>(*idx); - let mut children_zs = [F::ZERO; 4]; - for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; - } - let z = FWrap(self.poseidon_cache.hash4(&children_zs)); - self.z_cache.insert(*ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(*ptr)); - z - } - } - RawPtr::Hash6(idx) => { - if let Some(z) = self.z_cache.get(ptr) { - *z - } else { - let children_ptrs = self.expect_raw_ptrs::<6>(*idx); - let mut children_zs = [F::ZERO; 6]; - for (idx, child_ptr) in children_ptrs.iter().enumerate() { - children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; - } - let z = FWrap(self.poseidon_cache.hash6(&children_zs)); - self.z_cache.insert(*ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(*ptr)); - z - } - } - RawPtr::Hash8(idx) => { + macro_rules! hash_raw { + ($hash:ident, $n:expr, $idx:expr) => {{ if let Some(z) = self.z_cache.get(ptr) { *z } else { - let children_ptrs = self.expect_raw_ptrs::<8>(*idx); - let mut children_zs = [F::ZERO; 8]; + let children_ptrs = self.expect_raw_ptrs::<$n>($idx); + let mut children_zs = [F::ZERO; $n]; for (idx, child_ptr) in children_ptrs.iter().enumerate() { children_zs[idx] = self.hash_raw_ptr_unsafe(child_ptr).0; } - let z = FWrap(self.poseidon_cache.hash8(&children_zs)); + let z = FWrap(self.poseidon_cache.$hash(&children_zs)); self.z_cache.insert(*ptr, Box::new(z)); self.inverse_z_cache.insert(z, Box::new(*ptr)); z } - } + }}; + } + match ptr { + RawPtr::Atom(idx) => FWrap(*self.expect_f(*idx)), + RawPtr::Hash3(idx) => hash_raw!(hash3, 3, *idx), + RawPtr::Hash4(idx) => hash_raw!(hash4, 4, *idx), + RawPtr::Hash6(idx) => hash_raw!(hash6, 6, *idx), + RawPtr::Hash8(idx) => hash_raw!(hash8, 8, *idx), } } @@ -1005,7 +969,7 @@ impl Store { } pub fn hash_ptr(&self, ptr: &Ptr) -> ZPtr { - ZPtr::from_parts(*ptr.tag(), self.hash_raw_ptr(ptr.pay()).0) + ZPtr::from_parts(*ptr.tag(), self.hash_raw_ptr(ptr.raw()).0) } /// Constructs a vector of scalars that correspond to tags and hashes computed @@ -1014,9 +978,9 @@ impl Store { ptrs.iter() .fold(Vec::with_capacity(2 * ptrs.len()), |mut acc, ptr| { let tag = ptr.tag().to_field(); - let pay = self.hash_raw_ptr(ptr.pay()).0; + let payload = self.hash_raw_ptr(ptr.raw()).0; acc.push(tag); - acc.push(pay); + acc.push(payload); acc }) } @@ -1044,7 +1008,7 @@ impl Store { self.inverse_z_cache .get(z) .cloned() - .unwrap_or_else(|| self.opaque(*z)) + .unwrap_or_else(|| self.intern_raw_atom(z.0)) } #[inline] @@ -1087,7 +1051,7 @@ impl Ptr { } Char => { if let Some(c) = self - .pay() + .raw() .get_atom() .map(|idx| store.expect_f(idx)) .and_then(F::to_char) @@ -1117,7 +1081,7 @@ impl Ptr { } } Num => { - if let Some(f) = self.pay().get_atom().map(|idx| store.expect_f(idx)) { + if let Some(f) = self.raw().get_atom().map(|idx| store.expect_f(idx)) { if let Some(u) = f.to_u64() { u.to_string() } else { @@ -1129,7 +1093,7 @@ impl Ptr { } U64 => { if let Some(u) = self - .pay() + .raw() .get_atom() .map(|idx| store.expect_f(idx)) .and_then(F::to_u64) @@ -1139,7 +1103,7 @@ impl Ptr { "".into() } } - Fun => match self.pay().get_hash8() { + Fun => match self.raw().get_hash8() { None => "".into(), Some(idx) => { if let Some([vars, body, _, _]) = store.fetch_ptrs::<8, 4>(idx) { @@ -1161,7 +1125,7 @@ impl Ptr { } } }, - Thunk => match self.pay().get_hash4() { + Thunk => match self.raw().get_hash4() { None => "".into(), Some(idx) => { if let Some([val, cont]) = store.fetch_ptrs::<4, 2>(idx) { @@ -1175,7 +1139,7 @@ impl Ptr { } } }, - Comm => match self.pay().get_atom() { + Comm => match self.raw().get_atom() { Some(idx) => { let f = store.expect_f(idx); if store.inverse_z_cache.get(&FWrap(*f)).is_some() { @@ -1186,7 +1150,7 @@ impl Ptr { } None => "".into(), }, - Cproc => match self.pay().get_hash4() { + Cproc => match self.raw().get_hash4() { None => "".into(), Some(idx) => { if let Some([cproc_name, args]) = store.fetch_ptrs::<4, 2>(idx) { @@ -1248,7 +1212,7 @@ impl Ptr { store: &Store, state: &State, ) -> String { - match self.pay().get_hash8() { + match self.raw().get_hash8() { None => format!(""), Some(idx) => { if let Some([a, cont, _, _]) = store.fetch_ptrs::<8, 4>(idx) { @@ -1271,7 +1235,7 @@ impl Ptr { store: &Store, state: &State, ) -> String { - match self.pay().get_hash8() { + match self.raw().get_hash8() { None => format!(""), Some(idx) => { if let Some([a, b, cont, _]) = store.fetch_ptrs::<8, 4>(idx) { @@ -1296,7 +1260,7 @@ impl Ptr { store: &Store, state: &State, ) -> String { - match self.pay().get_hash8() { + match self.raw().get_hash8() { None => format!(""), Some(idx) => { if let Some([a, b, c, cont]) = store.fetch_ptrs::<8, 4>(idx) { @@ -1315,3 +1279,261 @@ impl Ptr { } } } + +#[cfg(test)] +mod tests { + use ff::Field; + use pasta_curves::pallas::Scalar as Fr; + use proptest::prelude::*; + + use crate::{ + field::LurkField, + lem::Tag, + parser::position::Pos, + state::{initial_lurk_state, lurk_sym}, + syntax::Syntax, + tag::{ExprTag, Tag as TagTrait}, + Num, Symbol, + }; + + use super::{Ptr, RawPtr, Store}; + + #[test] + fn test_car_cdr() { + let store = Store::::default(); + + // empty list + let nil = store.intern_nil(); + let (car, cdr) = store.car_cdr(&nil).unwrap(); + assert_eq!((&car, &cdr), (&nil, &nil)); + + // regular cons + let one = store.num_u64(1); + let a = store.char('a'); + let one_a = store.cons(one, a); + let (car, cdr) = store.car_cdr(&one_a).unwrap(); + assert_eq!((&one, &a), (&car, &cdr)); + + // string + let abc = store.intern_string("abc"); + let bc = store.intern_string("bc"); + let (car, cdr) = store.car_cdr(&abc).unwrap(); + assert_eq!((&a, &bc), (&car, &cdr)); + + // empty string + let empty_str = store.intern_string(""); + let (car, cdr) = store.car_cdr(&empty_str).unwrap(); + assert_eq!((&nil, &empty_str), (&car, &cdr)); + } + + #[test] + fn test_list() { + let store = Store::::default(); + let state = initial_lurk_state(); + + // empty list + let list = store.list(vec![]); + let nil = store.intern_nil(); + assert_eq!(&list, &nil); + let (elts, non_nil) = store.fetch_list(&list).unwrap(); + assert!(elts.is_empty()); + assert!(non_nil.is_none()); + + // proper list + let a = store.char('a'); + let b = store.char('b'); + let list = store.list(vec![a, b]); + assert_eq!(list.fmt_to_string(&store, state), "('a' 'b')"); + let (elts, non_nil) = store.fetch_list(&list).unwrap(); + assert_eq!(elts.len(), 2); + assert_eq!((&elts[0], &elts[1]), (&a, &b)); + assert!(non_nil.is_none()); + + // improper list + let c = store.char('c'); + let b_c = store.cons(b, c); + let a_b_c = store.cons(a, b_c); + let a_b_c_ = store.improper_list(vec![a, b], c); + assert_eq!(a_b_c, a_b_c_); + assert_eq!(a_b_c.fmt_to_string(&store, state), "('a' 'b' . 'c')"); + let (elts, non_nil) = store.fetch_list(&a_b_c).unwrap(); + assert_eq!(elts.len(), 2); + assert_eq!((&elts[0], &elts[1]), (&a, &b)); + assert_eq!(non_nil, Some(c)); + } + + #[test] + fn test_basic_hashing() { + let store = Store::::default(); + let zero = Fr::zero(); + let zero_tag = Tag::try_from(0).unwrap(); + let foo = store.intern_atom(zero_tag, zero); + + let z_foo = store.hash_ptr(&foo); + assert_eq!(z_foo.tag(), &zero_tag); + assert_eq!(z_foo.value(), &zero); + + let comm = store.hide(zero, foo); + assert_eq!(comm.tag(), &Tag::Expr(ExprTag::Comm)); + assert_eq!( + store.expect_f(comm.get_atom().unwrap()), + &store.poseidon_cache.hash3(&[zero; 3]) + ); + + let ptr2 = store.intern_2_ptrs(zero_tag, foo, foo); + let z_ptr2 = store.hash_ptr(&ptr2); + assert_eq!(z_ptr2.tag(), &zero_tag); + assert_eq!(z_ptr2.value(), &store.poseidon_cache.hash4(&[zero; 4])); + + let ptr3 = store.intern_3_ptrs(zero_tag, foo, foo, foo); + let z_ptr3 = store.hash_ptr(&ptr3); + assert_eq!(z_ptr3.tag(), &zero_tag); + assert_eq!(z_ptr3.value(), &store.poseidon_cache.hash6(&[zero; 6])); + + let ptr4 = store.intern_4_ptrs(zero_tag, foo, foo, foo, foo); + let z_ptr4 = store.hash_ptr(&ptr4); + assert_eq!(z_ptr4.tag(), &zero_tag); + assert_eq!(z_ptr4.value(), &store.poseidon_cache.hash8(&[zero; 8])); + } + + #[test] + fn test_display_opaque_knowledge() { + // bob creates a list + let store = Store::::default(); + let one = store.num_u64(1); + let two = store.num_u64(2); + let one_two = store.cons(one, two); + let hi = store.intern_string("hi"); + let z1 = store.hash_ptr(&hi); + let z2 = store.hash_ptr(&one_two); + let list = store.list(vec![one_two, hi]); + let z_list = store.hash_ptr(&list); + + // alice uses the hashed elements of the list to show that she + // can produce the same hash as the original z_list + let store = Store::::default(); + let a1 = store.opaque(z1); + let a2 = store.opaque(z2); + let list1 = store.list(vec![a1, a2]); + let list2 = store.list(vec![a2, a1]); + let z_list1 = store.hash_ptr(&list1); + let z_list2 = store.hash_ptr(&list2); + + // one of those lists should match the original + assert!(z_list == z_list1 || z_list == z_list2); + } + + #[test] + fn test_ptr_hashing_safety() { + let string = String::from_utf8(vec![b'0'; 4096]).unwrap(); + let store = Store::::default(); + let ptr = store.intern_string(&string); + // `hash_ptr_unsafe` would overflow the stack, whereas `hash_ptr` works + let x = store.hash_raw_ptr(ptr.raw()); + + let store = Store::::default(); + let ptr = store.intern_string(&string); + store.hydrate_z_cache(); + // but `hash_ptr_unsafe` works just fine after manual hydration + let y = store.hash_raw_ptr_unsafe(ptr.raw()); + + // and, of course, those functions result on the same `ZPtr` + assert_eq!(x, y); + } + + #[test] + fn string_hashing() { + let s = &Store::::default(); + let hi_ptr = s.intern_string("hi"); + + let hi_hash_manual = s.poseidon_cache.hash4(&[ + ExprTag::Char.to_field(), + Fr::from_char('h'), + ExprTag::Str.to_field(), + s.poseidon_cache.hash4(&[ + ExprTag::Char.to_field(), + Fr::from_char('i'), + ExprTag::Str.to_field(), + Fr::ZERO, + ]), + ]); + + let hi_hash = s.hash_ptr(&hi_ptr).1; + assert_eq!(hi_hash, hi_hash_manual); + } + + #[test] + fn symbol_hashing() { + let s = &Store::::default(); + let foo_ptr = s.intern_string("foo"); + let bar_ptr = s.intern_string("bar"); + let foo_bar_ptr = s.intern_symbol(&Symbol::sym_from_vec(vec!["foo".into(), "bar".into()])); + + let foo_z_ptr = s.hash_ptr(&foo_ptr); + let bar_z_ptr = s.hash_ptr(&bar_ptr); + + let foo_bar_hash_manual = s.poseidon_cache.hash4(&[ + ExprTag::Str.to_field(), + bar_z_ptr.1, + ExprTag::Sym.to_field(), + s.poseidon_cache.hash4(&[ + ExprTag::Str.to_field(), + foo_z_ptr.1, + ExprTag::Sym.to_field(), + Fr::ZERO, + ]), + ]); + + let foo_bar_hash = s.hash_ptr(&foo_bar_ptr).1; + assert_eq!(foo_bar_hash, foo_bar_hash_manual); + } + + // helper function to test syntax interning roundtrip + fn fetch_syntax(ptr: Ptr, store: &Store) -> Syntax { + match ptr.parts() { + (Tag::Expr(ExprTag::Num), RawPtr::Atom(idx)) => { + Syntax::Num(Pos::No, Num::Scalar(*store.expect_f(*idx))) + } + (Tag::Expr(ExprTag::Char), RawPtr::Atom(idx)) => { + Syntax::Char(Pos::No, store.expect_f(*idx).to_char().unwrap()) + } + (Tag::Expr(ExprTag::U64), RawPtr::Atom(idx)) => Syntax::UInt( + Pos::No, + crate::UInt::U64(store.expect_f(*idx).to_u64_unchecked()), + ), + (Tag::Expr(ExprTag::Sym | ExprTag::Key), RawPtr::Atom(_) | RawPtr::Hash4(_)) => { + Syntax::Symbol(Pos::No, store.fetch_symbol(&ptr).unwrap().into()) + } + (Tag::Expr(ExprTag::Str), RawPtr::Atom(_) | RawPtr::Hash4(_)) => { + Syntax::String(Pos::No, store.fetch_string(&ptr).unwrap()) + } + (Tag::Expr(ExprTag::Cons), RawPtr::Hash4(_)) => { + let (elts, last) = store.fetch_list(&ptr).unwrap(); + let elts = elts + .into_iter() + .map(|e| fetch_syntax(e, store)) + .collect::>(); + if let Some(last) = last { + Syntax::Improper(Pos::No, elts, fetch_syntax(last, store).into()) + } else { + Syntax::List(Pos::No, elts) + } + } + (Tag::Expr(ExprTag::Nil), RawPtr::Hash4(_)) => { + Syntax::Symbol(Pos::No, lurk_sym("nil").into()) + } + _ => unreachable!(), + } + } + + proptest! { + #[test] + fn syntax_roundtrip(x in any::>()) { + let store = Store::::default(); + let ptr1 = store.intern_syntax(x); + let y = fetch_syntax(ptr1, &store); + let ptr2 = store.intern_syntax(y); + assert_eq!(ptr1, ptr2); + } + } +} From f72246ad97195e1907ccfc5aa823885323c90d1e Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Tue, 2 Jan 2024 20:42:01 -0300 Subject: [PATCH 07/12] Updated some documentation --- src/lem/pointers.rs | 34 ++++++++++++++++------------------ src/lem/store.rs | 40 +++++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/src/lem/pointers.rs b/src/lem/pointers.rs index e211b0c7da..b8526980be 100644 --- a/src/lem/pointers.rs +++ b/src/lem/pointers.rs @@ -7,6 +7,9 @@ use crate::{ use super::Tag; +/// `RawPtr` is the basic pointer type of the LEM store. An `Atom` points to a field +/// element, and a `HashN` points to `N` children, which are also raw pointers. Thus, +/// they are a building block for graphs that represent Lurk data. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] pub enum RawPtr { Atom(usize), @@ -66,17 +69,9 @@ impl RawPtr { } } -/// `Ptr` is the main piece of data LEMs operate on. We can think of a pointer -/// as a building block for trees that represent Lurk data. A pointer can be a -/// atom that contains data encoded as an element of a `LurkField` or it can have -/// children. For performance, the children of a pointer are stored on an -/// `IndexSet` and the resulding index is carried by the pointer itself. -/// -/// A pointer also has a tag, which says what kind of data it encodes. On -/// previous implementations, the tag would be used to infer the number of -/// children a pointer has. However, LEMs require extra flexibility because LEM -/// hashing operations can plug any tag to the resulting pointer. Thus, the -/// number of children have to be made explicit as the `Ptr` enum. +/// `Ptr` is a tagged pointer. The tag is there to say what kind of data it encodes. +/// Since tags can be encoded as field elements, they are also able to be expressed +/// as raw pointers. A `Ptr` can thus be seen as a tuple of `RawPtr`s. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] pub struct Ptr { tag: Tag, @@ -179,15 +174,18 @@ impl Ptr { } } -/// A `ZPtr` is the result of "hydrating" a `Ptr`. This process is better -/// explained in the store but, in short, we want to know the Poseidon hash of -/// the children of a `Ptr`. +/// A `ZPtr` is the result of "hydrating" a `Ptr`, which is a process of replacing +/// indices by hashes. That is, a `ZPtr` is a content-addressed, tagged, pointer. +/// By analogy, we can view ordinary field elements as hydrated raw pointers. /// -/// `ZPtr`s are used mainly for proofs, but they're also useful when we want -/// to content-address a store. +/// With `ZPtr`s we are able to content-address arbitrary DAGs, and thus be able to +/// represent these data structures as field elements. This is how we can prove facts +/// about data structures only using field elements. `ZPtr`s are also useful when we +/// want to content-address the store. /// -/// An important note is that computing the respective `ZPtr` of a `Ptr` can be -/// expensive because of the Poseidon hashes. That's why we operate on `Ptr`s +/// In principle, `ZPtr`s could be used in place of `Ptr`, but it is important to +/// note that content-addressing can be expensive, especially in the context of +/// interpretation, because of the Poseidon hashes. That's why we operate on `Ptr`s /// when interpreting LEMs and delay the need for `ZPtr`s as much as possible. pub type ZPtr = crate::z_data::z_ptr::ZPtr; diff --git a/src/lem/store.rs b/src/lem/store.rs index 5a30c43d8a..37c0471d74 100644 --- a/src/lem/store.rs +++ b/src/lem/store.rs @@ -29,6 +29,16 @@ use crate::{ use super::pointers::{Ptr, RawPtr, ZPtr}; +/// The `Store` is a crucial part of Lurk's implementation and tries to be a +/// vesatile data structure for many parts of Lurk's data pipeline. +/// +/// It holds Lurk data structured as graphs of `RawPtr`s. When a `RawPtr` has +/// children, we store them in its respective `IndexSet`. These data structures +/// speed up LEM interpretation because lookups by indices are fast, and leave +/// all the hashing to be done by the hydration step in multiple threads. +/// +/// The `Store` also provides an infra to speed up interning strings and symbols. +/// This data is saved in `string_ptr_cache` and `symbol_ptr_cache`. #[derive(Debug)] pub struct Store { f_elts: FrozenIndexSet>>, @@ -849,7 +859,7 @@ impl Store { /// /// Warning: without cache hits, this function might blow up Rust's recursion /// depth limit. This limitation is circumvented by calling `hydrate_z_cache` - /// beforehand or by using `hash_ptr` instead, which is slightly slower. + /// beforehand or by using `hash_raw_ptr` instead, which is slightly slower. fn hash_raw_ptr_unsafe(&self, ptr: &RawPtr) -> FWrap { macro_rules! hash_raw { ($hash:ident, $n:expr, $idx:expr) => {{ @@ -880,8 +890,8 @@ impl Store { /// Hashes pointers in parallel, consuming chunks of length 256, which is a /// reasonably safe limit. The danger of longer chunks is that the rightmost /// pointers are the ones which are more likely to reach the recursion depth - /// limit in `hash_ptr_unsafe`. So we move in smaller chunks from left to - /// right, populating the `z_cache`, which can rescue `hash_ptr_unsafe` from + /// limit in `hash_raw_ptr_unsafe`. So we move in smaller chunks from left to + /// right, populating the `z_cache`, which can rescue `hash_raw_ptr_unsafe` from /// dangerously deep recursions fn hydrate_z_cache_with_ptrs(&self, ptrs: &[&RawPtr]) { ptrs.chunks(256).for_each(|chunk| { @@ -891,8 +901,8 @@ impl Store { }); } - /// Hashes enqueued `Ptr` trees from the bottom to the top, avoiding deep - /// recursions in `hash_ptr`. Resets the `dehydrated` queue afterwards. + /// Hashes enqueued `RawPtr` trees from the bottom to the top, avoiding deep + /// recursions in `hash_raw_ptr`. Resets the `dehydrated` queue afterwards. pub fn hydrate_z_cache(&self) { self.hydrate_z_cache_with_ptrs(&self.dehydrated.load().iter().collect::>()); self.dehydrated.swap(Arc::new(FrozenVec::default())); @@ -911,12 +921,12 @@ impl Store { } } - /// Safe version of `hash_ptr_unsafe` that doesn't hit a stack overflow by + /// Safe version of `hash_raw_ptr_unsafe` that doesn't hit a stack overflow by /// precomputing the pointers that need to be hashed in order to hash the /// provided `ptr` pub fn hash_raw_ptr(&self, ptr: &RawPtr) -> FWrap { if self.is_below_safe_threshold() { - // just run `hash_ptr_unsafe` for extra speed when the dehydrated + // just run `hash_raw_ptr_unsafe` for extra speed when the dehydrated // queue is small enough return self.hash_raw_ptr_unsafe(ptr); } @@ -964,10 +974,12 @@ impl Store { } ptrs.reverse(); self.hydrate_z_cache_with_ptrs(&ptrs.into_iter().collect::>()); - // Now it's okay to call `hash_ptr_unsafe` + // Now it's okay to call `hash_raw_ptr_unsafe` self.hash_raw_ptr_unsafe(ptr) } + /// Hydrates a `Ptr`. That is, creates a `ZPtr` with the tag of the pointer + /// and the hash of its raw pointer pub fn hash_ptr(&self, ptr: &Ptr) -> ZPtr { ZPtr::from_parts(*ptr.tag(), self.hash_raw_ptr(ptr.raw()).0) } @@ -1000,9 +1012,9 @@ impl Store { self.hash_ptr(a) == self.hash_ptr(b) } - /// Attempts to recover the `Ptr` that corresponds to `z_ptr` from - /// `inverse_z_cache`. If the mapping is not there, returns an atom pointer - /// with the same tag and value + /// Attempts to recover the `RawPtr` that corresponds to a field element `z` + /// from `inverse_z_cache`. If the mapping is not there, returns a raw atom + /// pointer with value #[inline] pub fn to_raw_ptr(&self, z: &FWrap) -> RawPtr { self.inverse_z_cache @@ -1011,6 +1023,8 @@ impl Store { .unwrap_or_else(|| self.intern_raw_atom(z.0)) } + /// Attempts to recover the `Ptr` that corresponds to a `ZPtr`. If the mapping + /// is not there, returns an atom pointer with the same tag and value #[inline] pub fn to_ptr(&self, z_ptr: &ZPtr) -> Ptr { Ptr::new(*z_ptr.tag(), self.to_raw_ptr(&FWrap(*z_ptr.value()))) @@ -1428,13 +1442,13 @@ mod tests { let string = String::from_utf8(vec![b'0'; 4096]).unwrap(); let store = Store::::default(); let ptr = store.intern_string(&string); - // `hash_ptr_unsafe` would overflow the stack, whereas `hash_ptr` works + // `hash_raw_ptr_unsafe` would overflow the stack, whereas `hash_raw_ptr` works let x = store.hash_raw_ptr(ptr.raw()); let store = Store::::default(); let ptr = store.intern_string(&string); store.hydrate_z_cache(); - // but `hash_ptr_unsafe` works just fine after manual hydration + // but `hash_raw_ptr_unsafe` works just fine after manual hydration let y = store.hash_raw_ptr_unsafe(ptr.raw()); // and, of course, those functions result on the same `ZPtr` From 87d2756d13b9b85dfa099f3c206770b89789ce59 Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Wed, 3 Jan 2024 14:54:08 -0300 Subject: [PATCH 08/12] Utility macros --- src/cli/repl/meta_cmd.rs | 3 +- src/cli/zstore.rs | 35 ++++---- src/coprocessor/gadgets.rs | 25 +++--- src/lem/eval.rs | 7 +- src/lem/interpreter.rs | 17 ++-- src/lem/store.rs | 132 ++++++++++++------------------ src/lem/tests/nivc_steps.rs | 9 +- src/proof/tests/nova_tests_lem.rs | 7 +- 8 files changed, 113 insertions(+), 122 deletions(-) diff --git a/src/cli/repl/meta_cmd.rs b/src/cli/repl/meta_cmd.rs index 53c75afc57..e1f99dfcb6 100644 --- a/src/cli/repl/meta_cmd.rs +++ b/src/cli/repl/meta_cmd.rs @@ -19,6 +19,7 @@ use crate::{ eval::evaluate_with_env_and_cont, multiframe::MultiFrame, pointers::{Ptr, RawPtr, ZPtr}, + store::expect_ptrs, Tag, }, package::{Package, SymbolRef}, @@ -945,7 +946,7 @@ impl MetaCmd { io[0].fmt_to_string(&repl.store, &repl.state.borrow()) ) }; - let [pre_verify, post_verify] = &repl.store.expect_2_ptrs(*idx); + let [pre_verify, post_verify] = &expect_ptrs!(repl.store, 2, *idx); if pre_verify.is_nil() { bail!("Pre-verification predicate rejected the input") diff --git a/src/cli/zstore.rs b/src/cli/zstore.rs index 07351f4767..99bf11932e 100644 --- a/src/cli/zstore.rs +++ b/src/cli/zstore.rs @@ -6,7 +6,7 @@ use crate::{ field::{FWrap, LurkField}, lem::{ pointers::{Ptr, RawPtr, ZPtr}, - store::Store, + store::{expect_ptrs, intern_ptrs_hydrated, Store}, }, }; @@ -53,7 +53,7 @@ impl ZDag { unreachable!() } RawPtr::Hash4(idx) => { - let [a, b] = store.expect_2_ptrs(*idx); + let [a, b] = expect_ptrs!(store, 2, *idx); let a = self.populate_with(&a, store, cache); let b = self.populate_with(&b, store, cache); let z_ptr = ZPtr::from_parts( @@ -69,7 +69,7 @@ impl ZDag { z_ptr } RawPtr::Hash6(idx) => { - let [a, b, c] = store.expect_3_ptrs(*idx); + let [a, b, c] = expect_ptrs!(store, 3, *idx); let a = self.populate_with(&a, store, cache); let b = self.populate_with(&b, store, cache); let c = self.populate_with(&c, store, cache); @@ -88,7 +88,7 @@ impl ZDag { z_ptr } RawPtr::Hash8(idx) => { - let [a, b, c, d] = store.expect_4_ptrs(*idx); + let [a, b, c, d] = expect_ptrs!(store, 4, *idx); let a = self.populate_with(&a, store, cache); let b = self.populate_with(&b, store, cache); let c = self.populate_with(&c, store, cache); @@ -137,20 +137,20 @@ impl ZDag { Some(ZPtrType::Tuple2(z1, z2)) => { let ptr1 = self.populate_store(z1, store, cache)?; let ptr2 = self.populate_store(z2, store, cache)?; - store.intern_2_ptrs_hydrated(*z_ptr.tag(), ptr1, ptr2, *z_ptr) + intern_ptrs_hydrated!(store, *z_ptr.tag(), *z_ptr, ptr1, ptr2) } Some(ZPtrType::Tuple3(z1, z2, z3)) => { let ptr1 = self.populate_store(z1, store, cache)?; let ptr2 = self.populate_store(z2, store, cache)?; let ptr3 = self.populate_store(z3, store, cache)?; - store.intern_3_ptrs_hydrated(*z_ptr.tag(), ptr1, ptr2, ptr3, *z_ptr) + intern_ptrs_hydrated!(store, *z_ptr.tag(), *z_ptr, ptr1, ptr2, ptr3) } Some(ZPtrType::Tuple4(z1, z2, z3, z4)) => { let ptr1 = self.populate_store(z1, store, cache)?; let ptr2 = self.populate_store(z2, store, cache)?; let ptr3 = self.populate_store(z3, store, cache)?; let ptr4 = self.populate_store(z4, store, cache)?; - store.intern_4_ptrs_hydrated(*z_ptr.tag(), ptr1, ptr2, ptr3, ptr4, *z_ptr) + intern_ptrs_hydrated!(store, *z_ptr.tag(), *z_ptr, ptr1, ptr2, ptr3, ptr4) } }; cache.insert(*z_ptr, ptr); @@ -281,7 +281,11 @@ mod tests { use crate::{ field::LurkField, - lem::{pointers::Ptr, store::Store, Tag}, + lem::{ + pointers::Ptr, + store::{intern_ptrs, Store}, + Tag, + }, tag::{ContTag, ExprTag, Op1, Op2}, }; @@ -302,23 +306,26 @@ mod tests { } else { match rnd % 4 { 0 => store.intern_atom(tag, Fp::from_u64(rnd)), - 1 => store.intern_2_ptrs( + 1 => intern_ptrs!( + store, tag, rng_interner(rng, max_depth - 1, store), - rng_interner(rng, max_depth - 1, store), + rng_interner(rng, max_depth - 1, store) ), - 2 => store.intern_3_ptrs( + 2 => intern_ptrs!( + store, tag, rng_interner(rng, max_depth - 1, store), rng_interner(rng, max_depth - 1, store), - rng_interner(rng, max_depth - 1, store), + rng_interner(rng, max_depth - 1, store) ), - 3 => store.intern_4_ptrs( + 3 => intern_ptrs!( + store, tag, rng_interner(rng, max_depth - 1, store), rng_interner(rng, max_depth - 1, store), rng_interner(rng, max_depth - 1, store), - rng_interner(rng, max_depth - 1, store), + rng_interner(rng, max_depth - 1, store) ), _ => unreachable!(), } diff --git a/src/coprocessor/gadgets.rs b/src/coprocessor/gadgets.rs index 371a078b03..e983da74ce 100644 --- a/src/coprocessor/gadgets.rs +++ b/src/coprocessor/gadgets.rs @@ -12,7 +12,7 @@ use crate::{ lem::{ circuit::GlobalAllocator, pointers::{Ptr, ZPtr}, - store::Store, + store::{expect_ptrs, Store}, }, tag::{ExprTag, Tag}, }; @@ -178,7 +178,7 @@ pub(crate) fn deconstruct_tuple2>( ) -> Result<(AllocatedPtr, AllocatedPtr), SynthesisError> { let (a, b) = if not_dummy.get_value() == Some(true) { let idx = get_ptr(tuple, store)?.get_index2().expect("invalid Ptr"); - let [a, b] = &store.expect_2_ptrs(idx); + let [a, b] = &expect_ptrs!(store, 2, idx); (store.hash_ptr(a), store.hash_ptr(b)) } else { (ZPtr::dummy(), ZPtr::dummy()) @@ -221,7 +221,7 @@ pub(crate) fn deconstruct_tuple3>( ) -> Result<(AllocatedPtr, AllocatedPtr, AllocatedPtr), SynthesisError> { let (a, b, c) = if not_dummy.get_value() == Some(true) { let idx = get_ptr(tuple, store)?.get_index3().expect("invalid Ptr"); - let [a, b, c] = &store.expect_3_ptrs(idx); + let [a, b, c] = &expect_ptrs!(store, 3, idx); (store.hash_ptr(a), store.hash_ptr(b), store.hash_ptr(c)) } else { (ZPtr::dummy(), ZPtr::dummy(), ZPtr::dummy()) @@ -275,7 +275,7 @@ pub(crate) fn deconstruct_tuple4>( > { let (a, b, c, d) = if not_dummy.get_value() == Some(true) { let idx = get_ptr(tuple, store)?.get_index4().expect("invalid Ptr"); - let [a, b, c, d] = &store.expect_4_ptrs(idx); + let [a, b, c, d] = &expect_ptrs!(store, 4, idx); ( store.hash_ptr(a), store.hash_ptr(b), @@ -538,7 +538,10 @@ mod test { deconstruct_tuple4, }, field::LurkField, - lem::{circuit::GlobalAllocator, store::Store}, + lem::{ + circuit::GlobalAllocator, + store::{intern_ptrs, Store}, + }, }; use super::{a_ptr_as_z_ptr, chain_car_cdr, construct_list, deconstruct_tuple2}; @@ -561,7 +564,7 @@ mod test { &a_nil, ) .unwrap(); - let nil2_ptr = store.intern_2_ptrs(*nil_tag, nil, nil); + let nil2_ptr = intern_ptrs!(store, *nil_tag, nil, nil); let z_nil2_ptr = store.hash_ptr(&nil2_ptr); assert_eq!(a_ptr_as_z_ptr(&nil2), Some(z_nil2_ptr)); @@ -575,7 +578,7 @@ mod test { &a_nil, ) .unwrap(); - let nil3_ptr = store.intern_3_ptrs(*nil_tag, nil, nil, nil); + let nil3_ptr = intern_ptrs!(store, *nil_tag, nil, nil, nil); let z_nil3_ptr = store.hash_ptr(&nil3_ptr); assert_eq!(a_ptr_as_z_ptr(&nil3), Some(z_nil3_ptr)); @@ -590,7 +593,7 @@ mod test { &a_nil, ) .unwrap(); - let nil4_ptr = store.intern_4_ptrs(*nil_tag, nil, nil, nil, nil); + let nil4_ptr = intern_ptrs!(store, *nil_tag, nil, nil, nil, nil); let z_nil4_ptr = store.hash_ptr(&nil4_ptr); assert_eq!(a_ptr_as_z_ptr(&nil4), Some(z_nil4_ptr)); } @@ -624,7 +627,7 @@ mod test { let nil_tag = *nil.tag(); let not_dummy = Boolean::Constant(true); - let tuple2 = store.intern_2_ptrs(nil_tag, nil, nil); + let tuple2 = intern_ptrs!(store, nil_tag, nil, nil); let z_tuple2 = store.hash_ptr(&tuple2); let a_tuple2 = AllocatedPtr::alloc_infallible(&mut cs.namespace(|| "tuple2"), || z_tuple2); let (a, b) = deconstruct_tuple2( @@ -637,7 +640,7 @@ mod test { assert_eq!(a_ptr_as_z_ptr(&a), Some(z_nil)); assert_eq!(a_ptr_as_z_ptr(&b), Some(z_nil)); - let tuple3 = store.intern_3_ptrs(nil_tag, nil, nil, nil); + let tuple3 = intern_ptrs!(store, nil_tag, nil, nil, nil); let z_tuple3 = store.hash_ptr(&tuple3); let a_tuple3 = AllocatedPtr::alloc_infallible(&mut cs.namespace(|| "tuple3"), || z_tuple3); let (a, b, c) = deconstruct_tuple3( @@ -651,7 +654,7 @@ mod test { assert_eq!(a_ptr_as_z_ptr(&b), Some(z_nil)); assert_eq!(a_ptr_as_z_ptr(&c), Some(z_nil)); - let tuple4 = store.intern_4_ptrs(nil_tag, nil, nil, nil, nil); + let tuple4 = intern_ptrs!(store, nil_tag, nil, nil, nil, nil); let z_tuple4 = store.hash_ptr(&tuple4); let a_tuple4 = AllocatedPtr::alloc_infallible(&mut cs.namespace(|| "tuple4"), || z_tuple4); let (a, b, c, d) = deconstruct_tuple4( diff --git a/src/lem/eval.rs b/src/lem/eval.rs index 9ad14daca6..e15afee04a 100644 --- a/src/lem/eval.rs +++ b/src/lem/eval.rs @@ -22,7 +22,7 @@ use crate::{ use super::{ interpreter::{Frame, Hints}, pointers::{Ptr, RawPtr}, - store::Store, + store::{fetch_ptrs, Store}, Ctrl, Func, Op, Tag, Var, }; @@ -42,9 +42,8 @@ fn get_pc>( ) -> usize { match expr.parts() { (Tag::Expr(Cproc), RawPtr::Hash4(idx)) => { - let [cproc, _] = &store - .fetch_2_ptrs(*idx) - .expect("Coprocessor expression is not interned"); + let [cproc, _] = + &fetch_ptrs!(store, 2, *idx).expect("Coprocessor expression is not interned"); let cproc_sym = store .fetch_symbol(cproc) .expect("Coprocessor expression is not interned"); diff --git a/src/lem/interpreter.rs b/src/lem/interpreter.rs index aa01851be4..1a9fdcfc13 100644 --- a/src/lem/interpreter.rs +++ b/src/lem/interpreter.rs @@ -4,7 +4,7 @@ use super::{ path::Path, pointers::{Ptr, RawPtr}, slot::{SlotData, Val}, - store::Store, + store::{fetch_ptrs, intern_ptrs, Store}, var_map::VarMap, Block, Ctrl, Func, Op, Tag, Var, }; @@ -341,7 +341,7 @@ impl Block { } Op::Cons2(img, tag, preimg) => { let preimg_ptrs = bindings.get_many_ptr(preimg)?; - let tgt_ptr = store.intern_2_ptrs(*tag, preimg_ptrs[0], preimg_ptrs[1]); + let tgt_ptr = intern_ptrs!(store, *tag, preimg_ptrs[0], preimg_ptrs[1]); bindings.insert_ptr(img.clone(), tgt_ptr); let vals = preimg_ptrs.into_iter().map(Val::Pointer).collect(); hints.hash4.push(Some(SlotData { vals })); @@ -349,19 +349,20 @@ impl Block { Op::Cons3(img, tag, preimg) => { let preimg_ptrs = bindings.get_many_ptr(preimg)?; let tgt_ptr = - store.intern_3_ptrs(*tag, preimg_ptrs[0], preimg_ptrs[1], preimg_ptrs[2]); + intern_ptrs!(store, *tag, preimg_ptrs[0], preimg_ptrs[1], preimg_ptrs[2]); bindings.insert_ptr(img.clone(), tgt_ptr); let vals = preimg_ptrs.into_iter().map(Val::Pointer).collect(); hints.hash6.push(Some(SlotData { vals })); } Op::Cons4(img, tag, preimg) => { let preimg_ptrs = bindings.get_many_ptr(preimg)?; - let tgt_ptr = store.intern_4_ptrs( + let tgt_ptr = intern_ptrs!( + store, *tag, preimg_ptrs[0], preimg_ptrs[1], preimg_ptrs[2], - preimg_ptrs[3], + preimg_ptrs[3] ); bindings.insert_ptr(img.clone(), tgt_ptr); let vals = preimg_ptrs.into_iter().map(Val::Pointer).collect(); @@ -372,7 +373,7 @@ impl Block { let Some(idx) = img_ptr.get_index2() else { bail!("{img} isn't a Tree2 pointer"); }; - let Some(preimg_ptrs) = store.fetch_2_ptrs(idx) else { + let Some(preimg_ptrs) = fetch_ptrs!(store, 2, idx) else { bail!("Couldn't fetch {img}'s children") }; for (var, ptr) in preimg.iter().zip(preimg_ptrs.iter()) { @@ -386,7 +387,7 @@ impl Block { let Some(idx) = img_ptr.get_index3() else { bail!("{img} isn't a Tree3 pointer"); }; - let Some(preimg_ptrs) = store.fetch_3_ptrs(idx) else { + let Some(preimg_ptrs) = fetch_ptrs!(store, 3, idx) else { bail!("Couldn't fetch {img}'s children") }; for (var, ptr) in preimg.iter().zip(preimg_ptrs.iter()) { @@ -400,7 +401,7 @@ impl Block { let Some(idx) = img_ptr.get_index4() else { bail!("{img} isn't a Tree4 pointer"); }; - let Some(preimg_ptrs) = store.fetch_4_ptrs(idx) else { + let Some(preimg_ptrs) = fetch_ptrs!(store, 4, idx) else { bail!("Couldn't fetch {img}'s children") }; for (var, ptr) in preimg.iter().zip(preimg_ptrs.iter()) { diff --git a/src/lem/store.rs b/src/lem/store.rs index 37c0471d74..e27b887f08 100644 --- a/src/lem/store.rs +++ b/src/lem/store.rs @@ -104,6 +104,44 @@ impl Default for Store { } } +// These are utility macros for store methods on `Ptr`s, especially because +// they contain two const generic variables (more on this later) +macro_rules! count { + () => (0); + ( $_x:tt $($xs:tt)* ) => (1 + crate::lem::store::count!($($xs)*)); +} +pub(crate) use count; + +macro_rules! intern_ptrs { + ($store:expr, $tag:expr, $($ptrs:expr),*) => {{ + const N: usize = crate::lem::store::count!($($ptrs)*); + ($store).intern_ptrs::<{2*N}, N>($tag, [$($ptrs),*]) + }} +} +pub(crate) use intern_ptrs; + +macro_rules! intern_ptrs_hydrated { + ($store:expr, $tag:expr, $z:expr, $($ptrs:expr),*) => {{ + const N: usize = crate::lem::store::count!($($ptrs)*); + ($store).intern_ptrs_hydrated::<{2*N}, N>($tag, [$($ptrs),*], $z) + }} +} +pub(crate) use intern_ptrs_hydrated; + +macro_rules! fetch_ptrs { + ($store:expr, $n:expr, $idx:expr) => {{ + ($store).fetch_ptrs::<{ 2 * $n }, $n>($idx) + }}; +} +pub(crate) use fetch_ptrs; + +macro_rules! expect_ptrs { + ($store:expr, $n:expr, $idx:expr) => {{ + ($store).expect_ptrs::<{ 2 * $n }, $n>($idx) + }}; +} +pub(crate) use expect_ptrs; + impl Store { /// Cost of poseidon hash with arity 3, including the input #[inline] @@ -157,7 +195,7 @@ impl Store { /// Since the `generic_const_exprs` feature is still unstable, we cannot substitute `N * 2` /// for generic const `P` and remove it completely, so we must keep it and do a dynamic assertion /// that it equals `N * 2`. This is not very ergonomic though, since we must add turbofishes - /// like `::<6, 3>` instead of the simpler `::<3>`. Could we maybe create a macro for these functions? + /// like `::<6, 3>` instead of the simpler `::<3>`. #[inline] pub fn ptrs_to_raw_ptrs(&self, ptrs: &[Ptr; P]) -> [RawPtr; N] { assert_eq!(P * 2, N); @@ -257,10 +295,10 @@ impl Store { &self, tag: Tag, ptrs: [Ptr; P], - z: FWrap, + z: ZPtr, ) -> Ptr { let raw_ptrs = self.ptrs_to_raw_ptrs::(&ptrs); - let payload = self.intern_raw_ptrs_hydrated::(raw_ptrs, z); + let payload = self.intern_raw_ptrs_hydrated::(raw_ptrs, FWrap(*z.value())); Ptr::new(tag, payload) } @@ -311,64 +349,6 @@ impl Store { .expect("Index missing from store") } - // TODO eventually deprecate these functions - #[inline] - pub fn intern_2_ptrs(&self, tag: Tag, a: Ptr, b: Ptr) -> Ptr { - self.intern_ptrs::<4, 2>(tag, [a, b]) - } - #[inline] - pub fn intern_3_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr) -> Ptr { - self.intern_ptrs::<6, 3>(tag, [a, b, c]) - } - #[inline] - pub fn intern_4_ptrs(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, d: Ptr) -> Ptr { - self.intern_ptrs::<8, 4>(tag, [a, b, c, d]) - } - #[inline] - pub fn intern_2_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, z: ZPtr) -> Ptr { - self.intern_ptrs_hydrated::<4, 2>(tag, [a, b], FWrap(*z.value())) - } - #[inline] - pub fn intern_3_ptrs_hydrated(&self, tag: Tag, a: Ptr, b: Ptr, c: Ptr, z: ZPtr) -> Ptr { - self.intern_ptrs_hydrated::<6, 3>(tag, [a, b, c], FWrap(*z.value())) - } - #[inline] - pub fn intern_4_ptrs_hydrated( - &self, - tag: Tag, - a: Ptr, - b: Ptr, - c: Ptr, - d: Ptr, - z: ZPtr, - ) -> Ptr { - self.intern_ptrs_hydrated::<8, 4>(tag, [a, b, c, d], FWrap(*z.value())) - } - #[inline] - pub fn fetch_2_ptrs(&self, idx: usize) -> Option<[Ptr; 2]> { - self.fetch_ptrs::<4, 2>(idx) - } - #[inline] - pub fn fetch_3_ptrs(&self, idx: usize) -> Option<[Ptr; 3]> { - self.fetch_ptrs::<6, 3>(idx) - } - #[inline] - pub fn fetch_4_ptrs(&self, idx: usize) -> Option<[Ptr; 4]> { - self.fetch_ptrs::<8, 4>(idx) - } - #[inline] - pub fn expect_2_ptrs(&self, idx: usize) -> [Ptr; 2] { - self.fetch_2_ptrs(idx).expect("Index missing from store") - } - #[inline] - pub fn expect_3_ptrs(&self, idx: usize) -> [Ptr; 3] { - self.fetch_3_ptrs(idx).expect("Index missing from store") - } - #[inline] - pub fn expect_4_ptrs(&self, idx: usize) -> [Ptr; 4] { - self.fetch_4_ptrs(idx).expect("Index missing from store") - } - #[inline] pub fn tag(&self, tag: Tag) -> RawPtr { self.intern_raw_atom(tag.to_field()) @@ -446,8 +426,7 @@ impl Store { } else { let empty_str = Ptr::new(Tag::Expr(Str), self.raw_zero()); let ptr = s.chars().rev().fold(empty_str, |acc, c| { - let ptrs = [self.char(c), acc]; - self.intern_ptrs::<4, 2>(Tag::Expr(Str), ptrs) + intern_ptrs!(self, Tag::Expr(Str), self.char(c), acc) }); self.string_ptr_cache.insert(s.to_string(), Box::new(ptr)); self.ptr_string_cache.insert(ptr, s.to_string()); @@ -496,8 +475,7 @@ impl Store { pub fn intern_symbol_path(&self, path: &[String]) -> Ptr { let zero_sym = Ptr::new(Tag::Expr(Sym), self.raw_zero()); path.iter().fold(zero_sym, |acc, s| { - let ptrs = [self.intern_string(s), acc]; - self.intern_ptrs::<4, 2>(Tag::Expr(Sym), ptrs) + intern_ptrs!(self, Tag::Expr(Sym), self.intern_string(s), acc) }) } @@ -673,14 +651,12 @@ impl Store { #[inline] pub fn cons(&self, car: Ptr, cdr: Ptr) -> Ptr { - let ptrs = [car, cdr]; - self.intern_ptrs::<4, 2>(Tag::Expr(Cons), ptrs) + intern_ptrs!(self, Tag::Expr(Cons), car, cdr) } #[inline] pub fn intern_fun(&self, arg: Ptr, body: Ptr, env: Ptr) -> Ptr { - let ptrs = [arg, body, env, self.dummy()]; - self.intern_ptrs::<8, 4>(Tag::Expr(Fun), ptrs) + intern_ptrs!(self, Tag::Expr(Fun), arg, body, env, self.dummy()) } #[inline] @@ -1120,7 +1096,7 @@ impl Ptr { Fun => match self.raw().get_hash8() { None => "".into(), Some(idx) => { - if let Some([vars, body, _, _]) = store.fetch_ptrs::<8, 4>(idx) { + if let Some([vars, body, _, _]) = fetch_ptrs!(store, 4, idx) { match vars.tag() { Tag::Expr(Nil) => { format!("", body.fmt_to_string(store, state)) @@ -1142,7 +1118,7 @@ impl Ptr { Thunk => match self.raw().get_hash4() { None => "".into(), Some(idx) => { - if let Some([val, cont]) = store.fetch_ptrs::<4, 2>(idx) { + if let Some([val, cont]) = fetch_ptrs!(store, 2, idx) { format!( "Thunk{{ value: {} => cont: {} }}", val.fmt_to_string(store, state), @@ -1167,7 +1143,7 @@ impl Ptr { Cproc => match self.raw().get_hash4() { None => "".into(), Some(idx) => { - if let Some([cproc_name, args]) = store.fetch_ptrs::<4, 2>(idx) { + if let Some([cproc_name, args]) = fetch_ptrs!(store, 2, idx) { format!( "", cproc_name.fmt_to_string(store, state), @@ -1229,7 +1205,7 @@ impl Ptr { match self.raw().get_hash8() { None => format!(""), Some(idx) => { - if let Some([a, cont, _, _]) = store.fetch_ptrs::<8, 4>(idx) { + if let Some([a, cont, _, _]) = fetch_ptrs!(store, 4, idx) { format!( "{name}{{ {field}: {}, continuation: {} }}", a.fmt_to_string(store, state), @@ -1252,7 +1228,7 @@ impl Ptr { match self.raw().get_hash8() { None => format!(""), Some(idx) => { - if let Some([a, b, cont, _]) = store.fetch_ptrs::<8, 4>(idx) { + if let Some([a, b, cont, _]) = fetch_ptrs!(store, 4, idx) { let (fa, fb) = fields; format!( "{name}{{ {fa}: {}, {fb}: {}, continuation: {} }}", @@ -1277,7 +1253,7 @@ impl Ptr { match self.raw().get_hash8() { None => format!(""), Some(idx) => { - if let Some([a, b, c, cont]) = store.fetch_ptrs::<8, 4>(idx) { + if let Some([a, b, c, cont]) = fetch_ptrs!(store, 4, idx) { let (fa, fb, fc) = fields; format!( "{name}{{ {fa}: {}, {fb}: {}, {fc}: {}, continuation: {} }}", @@ -1394,17 +1370,17 @@ mod tests { &store.poseidon_cache.hash3(&[zero; 3]) ); - let ptr2 = store.intern_2_ptrs(zero_tag, foo, foo); + let ptr2 = intern_ptrs!(store, zero_tag, foo, foo); let z_ptr2 = store.hash_ptr(&ptr2); assert_eq!(z_ptr2.tag(), &zero_tag); assert_eq!(z_ptr2.value(), &store.poseidon_cache.hash4(&[zero; 4])); - let ptr3 = store.intern_3_ptrs(zero_tag, foo, foo, foo); + let ptr3 = intern_ptrs!(store, zero_tag, foo, foo, foo); let z_ptr3 = store.hash_ptr(&ptr3); assert_eq!(z_ptr3.tag(), &zero_tag); assert_eq!(z_ptr3.value(), &store.poseidon_cache.hash6(&[zero; 6])); - let ptr4 = store.intern_4_ptrs(zero_tag, foo, foo, foo, foo); + let ptr4 = intern_ptrs!(store, zero_tag, foo, foo, foo, foo); let z_ptr4 = store.hash_ptr(&ptr4); assert_eq!(z_ptr4.tag(), &zero_tag); assert_eq!(z_ptr4.value(), &store.poseidon_cache.hash8(&[zero; 8])); diff --git a/src/lem/tests/nivc_steps.rs b/src/lem/tests/nivc_steps.rs index fe741e0d57..c1843d96c1 100644 --- a/src/lem/tests/nivc_steps.rs +++ b/src/lem/tests/nivc_steps.rs @@ -5,7 +5,7 @@ use crate::{ eval::lang::Lang, lem::{ eval::{evaluate, make_cprocs_funcs_from_lang, make_eval_step_from_config, EvalConfig}, - store::Store, + store::{expect_ptrs, intern_ptrs, Store}, Tag, }, state::user_sym, @@ -78,12 +78,13 @@ fn test_nivc_steps() { let expr = cproc_input.pop().unwrap(); let idx = expr.get_index2().unwrap(); - let [_, args] = store.expect_2_ptrs(idx); + let [_, args] = expect_ptrs!(store, 2, idx); let new_name = user_sym("cproc-dumb-not"); - let new_expr = store.intern_2_ptrs( + let new_expr = intern_ptrs!( + store, Tag::Expr(ExprTag::Cproc), store.intern_symbol(&new_name), - args, + args ); // `cproc` can't reduce the altered cproc input (with the wrong name) diff --git a/src/proof/tests/nova_tests_lem.rs b/src/proof/tests/nova_tests_lem.rs index 8867f589fd..b0ed617390 100644 --- a/src/proof/tests/nova_tests_lem.rs +++ b/src/proof/tests/nova_tests_lem.rs @@ -4,7 +4,10 @@ use std::{cell::RefCell, rc::Rc, sync::Arc}; use crate::{ eval::lang::{Coproc, Lang}, - lem::{store::Store, Tag}, + lem::{ + store::{intern_ptrs, Store}, + Tag, + }, num::Num, proof::nova::C1LEM, state::user_sym, @@ -3786,7 +3789,7 @@ fn test_prove_call_literal_fun() { let empty_env = s.intern_nil(); let args = s.list(vec![s.intern_user_symbol("x")]); let body = s.read_with_default_state("(+ x 1)").unwrap(); - let fun = s.intern_4_ptrs(Tag::Expr(ExprTag::Fun), args, body, empty_env, s.dummy()); + let fun = intern_ptrs!(s, Tag::Expr(ExprTag::Fun), args, body, empty_env, s.dummy()); let input = s.num_u64(9); let expr = s.list(vec![fun, input]); let res = s.num_u64(10); From c25d5fb4a679a44a7bb7f0097651c77281dd82d2 Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Wed, 3 Jan 2024 17:48:30 -0300 Subject: [PATCH 09/12] Reverted `Hash3` and `comms` change --- src/cli/repl/meta_cmd.rs | 2 +- src/cli/zstore.rs | 6 ------ src/lem/interpreter.rs | 8 +++---- src/lem/pointers.rs | 11 +--------- src/lem/store.rs | 46 +++++++--------------------------------- 5 files changed, 14 insertions(+), 59 deletions(-) diff --git a/src/cli/repl/meta_cmd.rs b/src/cli/repl/meta_cmd.rs index e1f99dfcb6..9b986a99d0 100644 --- a/src/cli/repl/meta_cmd.rs +++ b/src/cli/repl/meta_cmd.rs @@ -623,7 +623,7 @@ impl MetaCmd { .store .open(hash) .expect("data must have been committed"); - repl.hide(secret, fun) + repl.hide(*secret, *fun) }, }; } diff --git a/src/cli/zstore.rs b/src/cli/zstore.rs index 99bf11932e..1513d27f02 100644 --- a/src/cli/zstore.rs +++ b/src/cli/zstore.rs @@ -46,12 +46,6 @@ impl ZDag { self.0.insert(z_ptr, ZPtrType::Atom); z_ptr } - RawPtr::Hash3(_) => { - // `Hash3` is currently only used for commit caches, and in particular, they - // never appear as part of expressions, since commits are not pointers, but - // field elements - unreachable!() - } RawPtr::Hash4(idx) => { let [a, b] = expect_ptrs!(store, 2, *idx); let a = self.populate_with(&a, store, cache); diff --git a/src/lem/interpreter.rs b/src/lem/interpreter.rs index 1a9fdcfc13..10858b0035 100644 --- a/src/lem/interpreter.rs +++ b/src/lem/interpreter.rs @@ -431,13 +431,13 @@ impl Block { let Some((secret, ptr)) = store.open(hash) else { bail!("No committed data for hash {}", &hash.hex_digits()) }; - bindings.insert_ptr(tgt_ptr.clone(), ptr); + bindings.insert_ptr(tgt_ptr.clone(), *ptr); bindings.insert_ptr( tgt_secret.clone(), - store.intern_atom(Tag::Expr(Num), secret), + store.intern_atom(Tag::Expr(Num), *secret), ); - let secret_idx = store.intern_f(secret).0; - let vals = vec![Val::Num(secret_idx), Val::Pointer(ptr)]; + let secret_idx = store.intern_f(*secret).0; + let vals = vec![Val::Num(secret_idx), Val::Pointer(*ptr)]; hints.commitment.push(Some(SlotData { vals })); } Op::Unit(f) => f(), diff --git a/src/lem/pointers.rs b/src/lem/pointers.rs index b8526980be..d21432441c 100644 --- a/src/lem/pointers.rs +++ b/src/lem/pointers.rs @@ -13,7 +13,6 @@ use super::Tag; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] pub enum RawPtr { Atom(usize), - Hash3(usize), Hash4(usize), Hash6(usize), Hash8(usize), @@ -24,7 +23,7 @@ impl RawPtr { pub fn is_hash(&self) -> bool { matches!( self, - RawPtr::Hash3(..) | RawPtr::Hash4(..) | RawPtr::Hash6(..) | RawPtr::Hash8(..) + RawPtr::Hash4(..) | RawPtr::Hash6(..) | RawPtr::Hash8(..) ) } @@ -36,14 +35,6 @@ impl RawPtr { } } - #[inline] - pub fn get_hash3(&self) -> Option { - match self { - RawPtr::Hash3(x) => Some(*x), - _ => None, - } - } - #[inline] pub fn get_hash4(&self) -> Option { match self { diff --git a/src/lem/store.rs b/src/lem/store.rs index e27b887f08..0ad138e55c 100644 --- a/src/lem/store.rs +++ b/src/lem/store.rs @@ -42,7 +42,6 @@ use super::pointers::{Ptr, RawPtr, ZPtr}; #[derive(Debug)] pub struct Store { f_elts: FrozenIndexSet>>, - hash3: FrozenIndexSet>, hash4: FrozenIndexSet>, hash6: FrozenIndexSet>, hash8: FrozenIndexSet>, @@ -53,6 +52,8 @@ pub struct Store { ptr_string_cache: FrozenMap, ptr_symbol_cache: FrozenMap>, + comms: FrozenMap, Box<(F, Ptr)>>, // hash -> (secret, src) + pub poseidon_cache: PoseidonCache, pub inverse_poseidon_cache: InversePoseidonCache, @@ -83,7 +84,6 @@ impl Default for Store { Self { f_elts, - hash3: Default::default(), hash4: Default::default(), hash6: Default::default(), hash8: Default::default(), @@ -91,6 +91,7 @@ impl Default for Store { symbol_ptr_cache: Default::default(), ptr_string_cache: Default::default(), ptr_symbol_cache: Default::default(), + comms: Default::default(), poseidon_cache, inverse_poseidon_cache: Default::default(), dehydrated: Default::default(), @@ -273,7 +274,6 @@ impl Store { }}; } match N { - 3 => intern!(Hash3, hash3, 3), 4 => intern!(Hash4, hash4, 4), 6 => intern!(Hash6, hash6, 6), 8 => intern!(Hash8, hash8, 8), @@ -317,7 +317,6 @@ impl Store { }}; } match N { - 3 => fetch!(hash3, 3), 4 => fetch!(hash4, 4), 6 => fetch!(hash6, 6), 8 => fetch!(hash8, 8), @@ -599,21 +598,13 @@ impl Store { #[inline] pub fn add_comm(&self, hash: F, secret: F, payload: Ptr) { - let ptrs = [ - self.intern_raw_atom(secret), - self.tag(*payload.tag()), - *payload.raw(), - ]; - let (idx, _) = self.hash3.insert_probe(Box::new(ptrs)); - let ptr = RawPtr::Hash3(idx); - let z = FWrap(hash); - self.z_cache.insert(ptr, Box::new(z)); - self.inverse_z_cache.insert(z, Box::new(ptr)); + self.comms + .insert(FWrap::(hash), Box::new((secret, payload))); } #[inline] pub fn hide(&self, secret: F, payload: Ptr) -> Ptr { - self.comm(self.hide_ptr(secret, payload)) + self.comm(self.hide_and_return_z_payload(secret, payload).0) } pub fn hide_and_return_z_payload(&self, secret: F, payload: Ptr) -> (F, ZPtr) { @@ -631,22 +622,8 @@ impl Store { } #[inline] - pub fn open(&self, hash: F) -> Option<(F, Ptr)> { - let cached = self.inverse_z_cache.get(&FWrap(hash))?; - let [f, tag, pay] = self.fetch_raw_ptrs::<3>(cached.get_hash3()?)?; - let f = self.fetch_f(f.get_atom()?)?; - let ptr = self.raw_to_ptr(tag, pay)?; - Some((*f, ptr)) - } - - pub fn hide_ptr(&self, secret: F, payload: Ptr) -> F { - let hash = self.poseidon_cache.hash3(&[ - secret, - payload.tag().to_field(), - self.hash_raw_ptr(payload.raw()).0, - ]); - self.add_comm(hash, secret, payload); - hash + pub fn open(&self, hash: F) -> Option<&(F, Ptr)> { + self.comms.get(&FWrap(hash)) } #[inline] @@ -856,7 +833,6 @@ impl Store { } match ptr { RawPtr::Atom(idx) => FWrap(*self.expect_f(*idx)), - RawPtr::Hash3(idx) => hash_raw!(hash3, 3, *idx), RawPtr::Hash4(idx) => hash_raw!(hash4, 4, *idx), RawPtr::Hash6(idx) => hash_raw!(hash6, 6, *idx), RawPtr::Hash8(idx) => hash_raw!(hash8, 8, *idx), @@ -922,12 +898,6 @@ impl Store { while let Some(ptr) = stack.pop() { match ptr { RawPtr::Atom(..) => (), - RawPtr::Hash3(idx) => { - let ptrs = self.expect_raw_ptrs::<3>(*idx); - for ptr in ptrs { - feed_loop!(ptr) - } - } RawPtr::Hash4(idx) => { let ptrs = self.expect_raw_ptrs::<4>(*idx); for ptr in ptrs { From c13c50275eb6517f0982891f37d3d87af2963cd7 Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Thu, 4 Jan 2024 17:05:01 -0300 Subject: [PATCH 10/12] Tag interning optimization --- src/lem/mod.rs | 133 +++++++++++++++++++++++++++++++++++++++++++++-- src/lem/store.rs | 17 ++++-- 2 files changed, 142 insertions(+), 8 deletions(-) diff --git a/src/lem/mod.rs b/src/lem/mod.rs index 042386ad9b..e8b0b1ff72 100644 --- a/src/lem/mod.rs +++ b/src/lem/mod.rs @@ -155,10 +155,135 @@ impl Tag { #[inline] pub fn to_field(&self) -> F { match self { - Tag::Expr(tag) => tag.to_field(), - Tag::Cont(tag) => tag.to_field(), - Tag::Op1(tag) => tag.to_field(), - Tag::Op2(tag) => tag.to_field(), + Self::Expr(tag) => tag.to_field(), + Self::Cont(tag) => tag.to_field(), + Self::Op1(tag) => tag.to_field(), + Self::Op2(tag) => tag.to_field(), + } + } + + pub fn pos(i: usize) -> Option { + match i { + 0 => Some(Self::Expr(ExprTag::Nil)), + 1 => Some(Self::Expr(ExprTag::Cons)), + 2 => Some(Self::Expr(ExprTag::Sym)), + 3 => Some(Self::Expr(ExprTag::Fun)), + 4 => Some(Self::Expr(ExprTag::Num)), + 5 => Some(Self::Expr(ExprTag::Thunk)), + 6 => Some(Self::Expr(ExprTag::Str)), + 7 => Some(Self::Expr(ExprTag::Char)), + 8 => Some(Self::Expr(ExprTag::Comm)), + 9 => Some(Self::Expr(ExprTag::U64)), + 10 => Some(Self::Expr(ExprTag::Key)), + 11 => Some(Self::Expr(ExprTag::Cproc)), + 12 => Some(Self::Cont(ContTag::Outermost)), + 13 => Some(Self::Cont(ContTag::Call0)), + 14 => Some(Self::Cont(ContTag::Call)), + 15 => Some(Self::Cont(ContTag::Call2)), + 16 => Some(Self::Cont(ContTag::Tail)), + 17 => Some(Self::Cont(ContTag::Error)), + 18 => Some(Self::Cont(ContTag::Lookup)), + 19 => Some(Self::Cont(ContTag::Unop)), + 20 => Some(Self::Cont(ContTag::Binop)), + 21 => Some(Self::Cont(ContTag::Binop2)), + 22 => Some(Self::Cont(ContTag::If)), + 23 => Some(Self::Cont(ContTag::Let)), + 24 => Some(Self::Cont(ContTag::LetRec)), + 25 => Some(Self::Cont(ContTag::Dummy)), + 26 => Some(Self::Cont(ContTag::Terminal)), + 27 => Some(Self::Cont(ContTag::Emit)), + 28 => Some(Self::Cont(ContTag::Cproc)), + 29 => Some(Self::Op1(Op1::Car)), + 30 => Some(Self::Op1(Op1::Cdr)), + 31 => Some(Self::Op1(Op1::Atom)), + 32 => Some(Self::Op1(Op1::Emit)), + 33 => Some(Self::Op1(Op1::Open)), + 34 => Some(Self::Op1(Op1::Secret)), + 35 => Some(Self::Op1(Op1::Commit)), + 36 => Some(Self::Op1(Op1::Num)), + 37 => Some(Self::Op1(Op1::Comm)), + 38 => Some(Self::Op1(Op1::Char)), + 39 => Some(Self::Op1(Op1::Eval)), + 40 => Some(Self::Op1(Op1::U64)), + 41 => Some(Self::Op2(Op2::Sum)), + 42 => Some(Self::Op2(Op2::Diff)), + 43 => Some(Self::Op2(Op2::Product)), + 44 => Some(Self::Op2(Op2::Quotient)), + 45 => Some(Self::Op2(Op2::Equal)), + 46 => Some(Self::Op2(Op2::NumEqual)), + 47 => Some(Self::Op2(Op2::Less)), + 48 => Some(Self::Op2(Op2::Greater)), + 49 => Some(Self::Op2(Op2::LessEqual)), + 50 => Some(Self::Op2(Op2::GreaterEqual)), + 51 => Some(Self::Op2(Op2::Cons)), + 52 => Some(Self::Op2(Op2::StrCons)), + 53 => Some(Self::Op2(Op2::Begin)), + 54 => Some(Self::Op2(Op2::Hide)), + 55 => Some(Self::Op2(Op2::Modulo)), + 56 => Some(Self::Op2(Op2::Eval)), + _ => None, + } + } + + pub fn index(&self) -> usize { + match self { + Self::Expr(ExprTag::Nil) => 0, + Self::Expr(ExprTag::Cons) => 1, + Self::Expr(ExprTag::Sym) => 2, + Self::Expr(ExprTag::Fun) => 3, + Self::Expr(ExprTag::Num) => 4, + Self::Expr(ExprTag::Thunk) => 5, + Self::Expr(ExprTag::Str) => 6, + Self::Expr(ExprTag::Char) => 7, + Self::Expr(ExprTag::Comm) => 8, + Self::Expr(ExprTag::U64) => 9, + Self::Expr(ExprTag::Key) => 10, + Self::Expr(ExprTag::Cproc) => 11, + Self::Cont(ContTag::Outermost) => 12, + Self::Cont(ContTag::Call0) => 13, + Self::Cont(ContTag::Call) => 14, + Self::Cont(ContTag::Call2) => 15, + Self::Cont(ContTag::Tail) => 16, + Self::Cont(ContTag::Error) => 17, + Self::Cont(ContTag::Lookup) => 18, + Self::Cont(ContTag::Unop) => 19, + Self::Cont(ContTag::Binop) => 20, + Self::Cont(ContTag::Binop2) => 21, + Self::Cont(ContTag::If) => 22, + Self::Cont(ContTag::Let) => 23, + Self::Cont(ContTag::LetRec) => 24, + Self::Cont(ContTag::Dummy) => 25, + Self::Cont(ContTag::Terminal) => 26, + Self::Cont(ContTag::Emit) => 27, + Self::Cont(ContTag::Cproc) => 28, + Self::Op1(Op1::Car) => 29, + Self::Op1(Op1::Cdr) => 30, + Self::Op1(Op1::Atom) => 31, + Self::Op1(Op1::Emit) => 32, + Self::Op1(Op1::Open) => 33, + Self::Op1(Op1::Secret) => 34, + Self::Op1(Op1::Commit) => 35, + Self::Op1(Op1::Num) => 36, + Self::Op1(Op1::Comm) => 37, + Self::Op1(Op1::Char) => 38, + Self::Op1(Op1::Eval) => 39, + Self::Op1(Op1::U64) => 40, + Self::Op2(Op2::Sum) => 41, + Self::Op2(Op2::Diff) => 42, + Self::Op2(Op2::Product) => 43, + Self::Op2(Op2::Quotient) => 44, + Self::Op2(Op2::Equal) => 45, + Self::Op2(Op2::NumEqual) => 46, + Self::Op2(Op2::Less) => 47, + Self::Op2(Op2::Greater) => 48, + Self::Op2(Op2::LessEqual) => 49, + Self::Op2(Op2::GreaterEqual) => 50, + Self::Op2(Op2::Cons) => 51, + Self::Op2(Op2::StrCons) => 52, + Self::Op2(Op2::Begin) => 53, + Self::Op2(Op2::Hide) => 54, + Self::Op2(Op2::Modulo) => 55, + Self::Op2(Op2::Eval) => 56, } } } diff --git a/src/lem/store.rs b/src/lem/store.rs index 0ad138e55c..9f732f2efc 100644 --- a/src/lem/store.rs +++ b/src/lem/store.rs @@ -24,7 +24,6 @@ use crate::{ Tail, Terminal, Unop, }, tag::ExprTag::{Char, Comm, Cons, Cproc, Fun, Key, Nil, Num, Str, Sym, Thunk, U64}, - tag::Tag as TagTrait, }; use super::pointers::{Ptr, RawPtr, ZPtr}; @@ -76,7 +75,17 @@ impl Default for Store { let hash6zeros = poseidon_cache.hash6(&[F::ZERO; 6]); let hash8zeros = poseidon_cache.hash8(&[F::ZERO; 8]); + // Since tags are used very often, we will allocate them at the beginning + // in order, so that we do not need to use the `f_elts` when we have a tag + // This is similar to the `hashNzeros` optimization let f_elts = FrozenIndexSet::default(); + let mut i = 0; + while let Some(tag) = Tag::pos(i) { + let (j, _) = f_elts.insert_probe(FWrap(tag.to_field()).into()); + // This is to make sure the indices are ordered + assert_eq!(i, j); + i += 1; + } let (hash3zeros_idx, _) = f_elts.insert_probe(FWrap(hash3zeros).into()); let (hash4zeros_idx, _) = f_elts.insert_probe(FWrap(hash4zeros).into()); let (hash6zeros_idx, _) = f_elts.insert_probe(FWrap(hash6zeros).into()); @@ -350,14 +359,14 @@ impl Store { #[inline] pub fn tag(&self, tag: Tag) -> RawPtr { - self.intern_raw_atom(tag.to_field()) + // Tags are interned in order, so their index is the store index + RawPtr::Atom(tag.index()) } #[inline] pub fn fetch_tag(&self, ptr: &RawPtr) -> Option { let idx = ptr.get_atom()?; - let f = self.fetch_f(idx)?; - TagTrait::from_field(f) + Tag::pos(idx) } pub fn raw_to_ptr(&self, tag: &RawPtr, raw: &RawPtr) -> Option { From 337ef0a5b57f780c5b31971a8add400ca07e2098 Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Thu, 4 Jan 2024 18:13:57 -0300 Subject: [PATCH 11/12] LEM tag own file and tests --- src/cli/repl/meta_cmd.rs | 2 +- src/cli/repl/mod.rs | 2 +- src/cli/zstore.rs | 2 +- src/coprocessor/mod.rs | 2 +- src/lem/interpreter.rs | 3 +- src/lem/macros.rs | 8 +- src/lem/mod.rs | 207 +----------------------- src/lem/tag.rs | 252 ++++++++++++++++++++++++++++++ src/proof/tests/nova_tests_lem.rs | 2 +- src/tag.rs | 12 +- 10 files changed, 274 insertions(+), 218 deletions(-) create mode 100644 src/lem/tag.rs diff --git a/src/cli/repl/meta_cmd.rs b/src/cli/repl/meta_cmd.rs index 9b986a99d0..50764616bb 100644 --- a/src/cli/repl/meta_cmd.rs +++ b/src/cli/repl/meta_cmd.rs @@ -20,7 +20,7 @@ use crate::{ multiframe::MultiFrame, pointers::{Ptr, RawPtr, ZPtr}, store::expect_ptrs, - Tag, + tag::Tag, }, package::{Package, SymbolRef}, proof::{ diff --git a/src/cli/repl/mod.rs b/src/cli/repl/mod.rs index 724d9d22a4..071469e61d 100644 --- a/src/cli/repl/mod.rs +++ b/src/cli/repl/mod.rs @@ -28,7 +28,7 @@ use crate::{ multiframe::MultiFrame, pointers::{Ptr, RawPtr}, store::Store, - Tag, + tag::Tag, }, parser, proof::{nova::NovaProver, Prover, RecursiveSNARKTrait}, diff --git a/src/cli/zstore.rs b/src/cli/zstore.rs index 1513d27f02..96a0ebad8d 100644 --- a/src/cli/zstore.rs +++ b/src/cli/zstore.rs @@ -278,7 +278,7 @@ mod tests { lem::{ pointers::Ptr, store::{intern_ptrs, Store}, - Tag, + tag::Tag, }, tag::{ContTag, ExprTag, Op1, Op2}, }; diff --git a/src/coprocessor/mod.rs b/src/coprocessor/mod.rs index beb97360d8..a3be4afd8f 100644 --- a/src/coprocessor/mod.rs +++ b/src/coprocessor/mod.rs @@ -113,7 +113,7 @@ pub(crate) mod test { use super::*; use crate::circuit::gadgets::constraints::{alloc_equal, mul}; - use crate::lem::{pointers::RawPtr, Tag as LEMTag}; + use crate::lem::{pointers::RawPtr, tag::Tag as LEMTag}; use crate::tag::{ExprTag, Tag}; use std::marker::PhantomData; diff --git a/src/lem/interpreter.rs b/src/lem/interpreter.rs index 10858b0035..a656a86122 100644 --- a/src/lem/interpreter.rs +++ b/src/lem/interpreter.rs @@ -5,8 +5,9 @@ use super::{ pointers::{Ptr, RawPtr}, slot::{SlotData, Val}, store::{fetch_ptrs, intern_ptrs, Store}, + tag::Tag, var_map::VarMap, - Block, Ctrl, Func, Op, Tag, Var, + Block, Ctrl, Func, Op, Var, }; use crate::{ diff --git a/src/lem/macros.rs b/src/lem/macros.rs index 7e16a6a720..bac1d1e748 100644 --- a/src/lem/macros.rs +++ b/src/lem/macros.rs @@ -30,16 +30,16 @@ macro_rules! lit { #[macro_export] macro_rules! tag { ( Expr::$tag:ident ) => { - $crate::lem::Tag::Expr($crate::tag::ExprTag::$tag) + $crate::lem::tag::Tag::Expr($crate::tag::ExprTag::$tag) }; ( Cont::$tag:ident ) => { - $crate::lem::Tag::Cont($crate::tag::ContTag::$tag) + $crate::lem::tag::Tag::Cont($crate::tag::ContTag::$tag) }; ( Op1::$tag:ident ) => { - $crate::lem::Tag::Op1($crate::tag::Op1::$tag) + $crate::lem::tag::Tag::Op1($crate::tag::Op1::$tag) }; ( Op2::$tag:ident ) => { - $crate::lem::Tag::Op2($crate::tag::Op2::$tag) + $crate::lem::tag::Tag::Op2($crate::tag::Op2::$tag) }; } diff --git a/src/lem/mod.rs b/src/lem/mod.rs index e8b0b1ff72..f6986c4bb1 100644 --- a/src/lem/mod.rs +++ b/src/lem/mod.rs @@ -68,23 +68,19 @@ mod path; pub mod pointers; mod slot; pub mod store; +pub mod tag; mod var_map; use anyhow::{bail, Result}; use indexmap::IndexMap; -use serde::{Deserialize, Serialize}; use std::sync::Arc; #[cfg(test)] mod tests; -use crate::{ - field::LurkField, - symbol::Symbol, - tag::{ContTag, ExprTag, Op1, Op2, Tag as TagTrait}, -}; +use crate::{field::LurkField, symbol::Symbol}; -use self::{pointers::Ptr, slot::SlotsCounter, store::Store, var_map::VarMap}; +use self::{pointers::Ptr, slot::SlotsCounter, store::Store, tag::Tag, var_map::VarMap}; pub type AString = Arc; @@ -103,203 +99,6 @@ pub struct Func { #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct Var(AString); -/// The LEM `Tag` is a wrapper around other types that are used as tags -#[derive(Copy, Debug, PartialEq, Clone, Eq, Hash, Serialize, Deserialize)] -pub enum Tag { - Expr(ExprTag), - Cont(ContTag), - Op1(Op1), - Op2(Op2), -} - -impl TryFrom for Tag { - type Error = anyhow::Error; - - fn try_from(val: u16) -> Result { - if let Ok(tag) = ExprTag::try_from(val) { - Ok(Tag::Expr(tag)) - } else if let Ok(tag) = ContTag::try_from(val) { - Ok(Tag::Cont(tag)) - } else if let Ok(tag) = Op1::try_from(val) { - Ok(Tag::Op1(tag)) - } else if let Ok(tag) = Op2::try_from(val) { - Ok(Tag::Op2(tag)) - } else { - bail!("Invalid u16 for Tag: {val}") - } - } -} - -impl From for u16 { - fn from(val: Tag) -> Self { - match val { - Tag::Expr(tag) => tag.into(), - Tag::Cont(tag) => tag.into(), - Tag::Op1(tag) => tag.into(), - Tag::Op2(tag) => tag.into(), - } - } -} - -impl TagTrait for Tag { - fn from_field(f: &F) -> Option { - Self::try_from(f.to_u16()?).ok() - } - - fn to_field(&self) -> F { - Tag::to_field(self) - } -} - -impl Tag { - #[inline] - pub fn to_field(&self) -> F { - match self { - Self::Expr(tag) => tag.to_field(), - Self::Cont(tag) => tag.to_field(), - Self::Op1(tag) => tag.to_field(), - Self::Op2(tag) => tag.to_field(), - } - } - - pub fn pos(i: usize) -> Option { - match i { - 0 => Some(Self::Expr(ExprTag::Nil)), - 1 => Some(Self::Expr(ExprTag::Cons)), - 2 => Some(Self::Expr(ExprTag::Sym)), - 3 => Some(Self::Expr(ExprTag::Fun)), - 4 => Some(Self::Expr(ExprTag::Num)), - 5 => Some(Self::Expr(ExprTag::Thunk)), - 6 => Some(Self::Expr(ExprTag::Str)), - 7 => Some(Self::Expr(ExprTag::Char)), - 8 => Some(Self::Expr(ExprTag::Comm)), - 9 => Some(Self::Expr(ExprTag::U64)), - 10 => Some(Self::Expr(ExprTag::Key)), - 11 => Some(Self::Expr(ExprTag::Cproc)), - 12 => Some(Self::Cont(ContTag::Outermost)), - 13 => Some(Self::Cont(ContTag::Call0)), - 14 => Some(Self::Cont(ContTag::Call)), - 15 => Some(Self::Cont(ContTag::Call2)), - 16 => Some(Self::Cont(ContTag::Tail)), - 17 => Some(Self::Cont(ContTag::Error)), - 18 => Some(Self::Cont(ContTag::Lookup)), - 19 => Some(Self::Cont(ContTag::Unop)), - 20 => Some(Self::Cont(ContTag::Binop)), - 21 => Some(Self::Cont(ContTag::Binop2)), - 22 => Some(Self::Cont(ContTag::If)), - 23 => Some(Self::Cont(ContTag::Let)), - 24 => Some(Self::Cont(ContTag::LetRec)), - 25 => Some(Self::Cont(ContTag::Dummy)), - 26 => Some(Self::Cont(ContTag::Terminal)), - 27 => Some(Self::Cont(ContTag::Emit)), - 28 => Some(Self::Cont(ContTag::Cproc)), - 29 => Some(Self::Op1(Op1::Car)), - 30 => Some(Self::Op1(Op1::Cdr)), - 31 => Some(Self::Op1(Op1::Atom)), - 32 => Some(Self::Op1(Op1::Emit)), - 33 => Some(Self::Op1(Op1::Open)), - 34 => Some(Self::Op1(Op1::Secret)), - 35 => Some(Self::Op1(Op1::Commit)), - 36 => Some(Self::Op1(Op1::Num)), - 37 => Some(Self::Op1(Op1::Comm)), - 38 => Some(Self::Op1(Op1::Char)), - 39 => Some(Self::Op1(Op1::Eval)), - 40 => Some(Self::Op1(Op1::U64)), - 41 => Some(Self::Op2(Op2::Sum)), - 42 => Some(Self::Op2(Op2::Diff)), - 43 => Some(Self::Op2(Op2::Product)), - 44 => Some(Self::Op2(Op2::Quotient)), - 45 => Some(Self::Op2(Op2::Equal)), - 46 => Some(Self::Op2(Op2::NumEqual)), - 47 => Some(Self::Op2(Op2::Less)), - 48 => Some(Self::Op2(Op2::Greater)), - 49 => Some(Self::Op2(Op2::LessEqual)), - 50 => Some(Self::Op2(Op2::GreaterEqual)), - 51 => Some(Self::Op2(Op2::Cons)), - 52 => Some(Self::Op2(Op2::StrCons)), - 53 => Some(Self::Op2(Op2::Begin)), - 54 => Some(Self::Op2(Op2::Hide)), - 55 => Some(Self::Op2(Op2::Modulo)), - 56 => Some(Self::Op2(Op2::Eval)), - _ => None, - } - } - - pub fn index(&self) -> usize { - match self { - Self::Expr(ExprTag::Nil) => 0, - Self::Expr(ExprTag::Cons) => 1, - Self::Expr(ExprTag::Sym) => 2, - Self::Expr(ExprTag::Fun) => 3, - Self::Expr(ExprTag::Num) => 4, - Self::Expr(ExprTag::Thunk) => 5, - Self::Expr(ExprTag::Str) => 6, - Self::Expr(ExprTag::Char) => 7, - Self::Expr(ExprTag::Comm) => 8, - Self::Expr(ExprTag::U64) => 9, - Self::Expr(ExprTag::Key) => 10, - Self::Expr(ExprTag::Cproc) => 11, - Self::Cont(ContTag::Outermost) => 12, - Self::Cont(ContTag::Call0) => 13, - Self::Cont(ContTag::Call) => 14, - Self::Cont(ContTag::Call2) => 15, - Self::Cont(ContTag::Tail) => 16, - Self::Cont(ContTag::Error) => 17, - Self::Cont(ContTag::Lookup) => 18, - Self::Cont(ContTag::Unop) => 19, - Self::Cont(ContTag::Binop) => 20, - Self::Cont(ContTag::Binop2) => 21, - Self::Cont(ContTag::If) => 22, - Self::Cont(ContTag::Let) => 23, - Self::Cont(ContTag::LetRec) => 24, - Self::Cont(ContTag::Dummy) => 25, - Self::Cont(ContTag::Terminal) => 26, - Self::Cont(ContTag::Emit) => 27, - Self::Cont(ContTag::Cproc) => 28, - Self::Op1(Op1::Car) => 29, - Self::Op1(Op1::Cdr) => 30, - Self::Op1(Op1::Atom) => 31, - Self::Op1(Op1::Emit) => 32, - Self::Op1(Op1::Open) => 33, - Self::Op1(Op1::Secret) => 34, - Self::Op1(Op1::Commit) => 35, - Self::Op1(Op1::Num) => 36, - Self::Op1(Op1::Comm) => 37, - Self::Op1(Op1::Char) => 38, - Self::Op1(Op1::Eval) => 39, - Self::Op1(Op1::U64) => 40, - Self::Op2(Op2::Sum) => 41, - Self::Op2(Op2::Diff) => 42, - Self::Op2(Op2::Product) => 43, - Self::Op2(Op2::Quotient) => 44, - Self::Op2(Op2::Equal) => 45, - Self::Op2(Op2::NumEqual) => 46, - Self::Op2(Op2::Less) => 47, - Self::Op2(Op2::Greater) => 48, - Self::Op2(Op2::LessEqual) => 49, - Self::Op2(Op2::GreaterEqual) => 50, - Self::Op2(Op2::Cons) => 51, - Self::Op2(Op2::StrCons) => 52, - Self::Op2(Op2::Begin) => 53, - Self::Op2(Op2::Hide) => 54, - Self::Op2(Op2::Modulo) => 55, - Self::Op2(Op2::Eval) => 56, - } - } -} - -impl std::fmt::Display for Tag { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use Tag::{Cont, Expr, Op1, Op2}; - match self { - Expr(tag) => write!(f, "expr.{}", tag), - Cont(tag) => write!(f, "cont.{}", tag), - Op1(tag) => write!(f, "op1.{}", tag), - Op2(tag) => write!(f, "op2.{}", tag), - } - } -} - /// LEM literals #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum Lit { diff --git a/src/lem/tag.rs b/src/lem/tag.rs new file mode 100644 index 0000000000..76c2e4f9ad --- /dev/null +++ b/src/lem/tag.rs @@ -0,0 +1,252 @@ +use anyhow::{bail, Result}; +use serde::{Deserialize, Serialize}; + +use crate::{ + field::LurkField, + tag::{ContTag, ExprTag, Op1, Op2, Tag as TagTrait}, +}; + +/// The LEM `Tag` is a wrapper around other types that are used as tags +#[derive(Copy, Debug, PartialEq, Clone, Eq, Hash, Serialize, Deserialize)] +pub enum Tag { + Expr(ExprTag), + Cont(ContTag), + Op1(Op1), + Op2(Op2), +} + +impl TryFrom for Tag { + type Error = anyhow::Error; + + fn try_from(val: u16) -> Result { + if let Ok(tag) = ExprTag::try_from(val) { + Ok(Tag::Expr(tag)) + } else if let Ok(tag) = ContTag::try_from(val) { + Ok(Tag::Cont(tag)) + } else if let Ok(tag) = Op1::try_from(val) { + Ok(Tag::Op1(tag)) + } else if let Ok(tag) = Op2::try_from(val) { + Ok(Tag::Op2(tag)) + } else { + bail!("Invalid u16 for Tag: {val}") + } + } +} + +impl From for u16 { + fn from(val: Tag) -> Self { + match val { + Tag::Expr(tag) => tag.into(), + Tag::Cont(tag) => tag.into(), + Tag::Op1(tag) => tag.into(), + Tag::Op2(tag) => tag.into(), + } + } +} + +impl TagTrait for Tag { + fn from_field(f: &F) -> Option { + Self::try_from(f.to_u16()?).ok() + } + + fn to_field(&self) -> F { + Tag::to_field(self) + } +} + +impl Tag { + #[inline] + pub fn to_field(&self) -> F { + match self { + Self::Expr(tag) => tag.to_field(), + Self::Cont(tag) => tag.to_field(), + Self::Op1(tag) => tag.to_field(), + Self::Op2(tag) => tag.to_field(), + } + } + + pub fn pos(i: usize) -> Option { + match i { + 0 => Some(Self::Expr(ExprTag::Nil)), + 1 => Some(Self::Expr(ExprTag::Cons)), + 2 => Some(Self::Expr(ExprTag::Sym)), + 3 => Some(Self::Expr(ExprTag::Fun)), + 4 => Some(Self::Expr(ExprTag::Num)), + 5 => Some(Self::Expr(ExprTag::Thunk)), + 6 => Some(Self::Expr(ExprTag::Str)), + 7 => Some(Self::Expr(ExprTag::Char)), + 8 => Some(Self::Expr(ExprTag::Comm)), + 9 => Some(Self::Expr(ExprTag::U64)), + 10 => Some(Self::Expr(ExprTag::Key)), + 11 => Some(Self::Expr(ExprTag::Cproc)), + 12 => Some(Self::Cont(ContTag::Outermost)), + 13 => Some(Self::Cont(ContTag::Call0)), + 14 => Some(Self::Cont(ContTag::Call)), + 15 => Some(Self::Cont(ContTag::Call2)), + 16 => Some(Self::Cont(ContTag::Tail)), + 17 => Some(Self::Cont(ContTag::Error)), + 18 => Some(Self::Cont(ContTag::Lookup)), + 19 => Some(Self::Cont(ContTag::Unop)), + 20 => Some(Self::Cont(ContTag::Binop)), + 21 => Some(Self::Cont(ContTag::Binop2)), + 22 => Some(Self::Cont(ContTag::If)), + 23 => Some(Self::Cont(ContTag::Let)), + 24 => Some(Self::Cont(ContTag::LetRec)), + 25 => Some(Self::Cont(ContTag::Dummy)), + 26 => Some(Self::Cont(ContTag::Terminal)), + 27 => Some(Self::Cont(ContTag::Emit)), + 28 => Some(Self::Cont(ContTag::Cproc)), + 29 => Some(Self::Op1(Op1::Car)), + 30 => Some(Self::Op1(Op1::Cdr)), + 31 => Some(Self::Op1(Op1::Atom)), + 32 => Some(Self::Op1(Op1::Emit)), + 33 => Some(Self::Op1(Op1::Open)), + 34 => Some(Self::Op1(Op1::Secret)), + 35 => Some(Self::Op1(Op1::Commit)), + 36 => Some(Self::Op1(Op1::Num)), + 37 => Some(Self::Op1(Op1::Comm)), + 38 => Some(Self::Op1(Op1::Char)), + 39 => Some(Self::Op1(Op1::Eval)), + 40 => Some(Self::Op1(Op1::U64)), + 41 => Some(Self::Op2(Op2::Sum)), + 42 => Some(Self::Op2(Op2::Diff)), + 43 => Some(Self::Op2(Op2::Product)), + 44 => Some(Self::Op2(Op2::Quotient)), + 45 => Some(Self::Op2(Op2::Equal)), + 46 => Some(Self::Op2(Op2::NumEqual)), + 47 => Some(Self::Op2(Op2::Less)), + 48 => Some(Self::Op2(Op2::Greater)), + 49 => Some(Self::Op2(Op2::LessEqual)), + 50 => Some(Self::Op2(Op2::GreaterEqual)), + 51 => Some(Self::Op2(Op2::Cons)), + 52 => Some(Self::Op2(Op2::StrCons)), + 53 => Some(Self::Op2(Op2::Begin)), + 54 => Some(Self::Op2(Op2::Hide)), + 55 => Some(Self::Op2(Op2::Modulo)), + 56 => Some(Self::Op2(Op2::Eval)), + _ => None, + } + } + + pub fn index(&self) -> usize { + match self { + Self::Expr(ExprTag::Nil) => 0, + Self::Expr(ExprTag::Cons) => 1, + Self::Expr(ExprTag::Sym) => 2, + Self::Expr(ExprTag::Fun) => 3, + Self::Expr(ExprTag::Num) => 4, + Self::Expr(ExprTag::Thunk) => 5, + Self::Expr(ExprTag::Str) => 6, + Self::Expr(ExprTag::Char) => 7, + Self::Expr(ExprTag::Comm) => 8, + Self::Expr(ExprTag::U64) => 9, + Self::Expr(ExprTag::Key) => 10, + Self::Expr(ExprTag::Cproc) => 11, + Self::Cont(ContTag::Outermost) => 12, + Self::Cont(ContTag::Call0) => 13, + Self::Cont(ContTag::Call) => 14, + Self::Cont(ContTag::Call2) => 15, + Self::Cont(ContTag::Tail) => 16, + Self::Cont(ContTag::Error) => 17, + Self::Cont(ContTag::Lookup) => 18, + Self::Cont(ContTag::Unop) => 19, + Self::Cont(ContTag::Binop) => 20, + Self::Cont(ContTag::Binop2) => 21, + Self::Cont(ContTag::If) => 22, + Self::Cont(ContTag::Let) => 23, + Self::Cont(ContTag::LetRec) => 24, + Self::Cont(ContTag::Dummy) => 25, + Self::Cont(ContTag::Terminal) => 26, + Self::Cont(ContTag::Emit) => 27, + Self::Cont(ContTag::Cproc) => 28, + Self::Op1(Op1::Car) => 29, + Self::Op1(Op1::Cdr) => 30, + Self::Op1(Op1::Atom) => 31, + Self::Op1(Op1::Emit) => 32, + Self::Op1(Op1::Open) => 33, + Self::Op1(Op1::Secret) => 34, + Self::Op1(Op1::Commit) => 35, + Self::Op1(Op1::Num) => 36, + Self::Op1(Op1::Comm) => 37, + Self::Op1(Op1::Char) => 38, + Self::Op1(Op1::Eval) => 39, + Self::Op1(Op1::U64) => 40, + Self::Op2(Op2::Sum) => 41, + Self::Op2(Op2::Diff) => 42, + Self::Op2(Op2::Product) => 43, + Self::Op2(Op2::Quotient) => 44, + Self::Op2(Op2::Equal) => 45, + Self::Op2(Op2::NumEqual) => 46, + Self::Op2(Op2::Less) => 47, + Self::Op2(Op2::Greater) => 48, + Self::Op2(Op2::LessEqual) => 49, + Self::Op2(Op2::GreaterEqual) => 50, + Self::Op2(Op2::Cons) => 51, + Self::Op2(Op2::StrCons) => 52, + Self::Op2(Op2::Begin) => 53, + Self::Op2(Op2::Hide) => 54, + Self::Op2(Op2::Modulo) => 55, + Self::Op2(Op2::Eval) => 56, + } + } +} + +impl std::fmt::Display for Tag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use Tag::{Cont, Expr, Op1, Op2}; + match self { + Expr(tag) => write!(f, "expr.{}", tag), + Cont(tag) => write!(f, "cont.{}", tag), + Op1(tag) => write!(f, "op1.{}", tag), + Op2(tag) => write!(f, "op2.{}", tag), + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::tag::{CONT_TAG_INIT, EXPR_TAG_INIT, OP1_TAG_INIT, OP2_TAG_INIT}; + + #[test] + fn pos_index_roundtrip() { + let mut i = 0; + while let Some(tag) = Tag::pos(i) { + let j = tag.index(); + assert_eq!(i, j); + i += 1; + } + + let mut i = EXPR_TAG_INIT; + while let Ok(expr_tag) = ExprTag::try_from(i) { + let tag = Tag::Expr(expr_tag); + let tag_2 = Tag::pos(tag.index()).unwrap(); + assert_eq!(tag, tag_2); + i += 1; + } + + let mut i = CONT_TAG_INIT; + while let Ok(cont_tag) = ContTag::try_from(i) { + let tag = Tag::Cont(cont_tag); + let tag_2 = Tag::pos(tag.index()).unwrap(); + assert_eq!(tag, tag_2); + i += 1; + } + + let mut i = OP1_TAG_INIT; + while let Ok(op1_tag) = Op1::try_from(i) { + let tag = Tag::Op1(op1_tag); + let tag_2 = Tag::pos(tag.index()).unwrap(); + assert_eq!(tag, tag_2); + i += 1; + } + + let mut i = OP2_TAG_INIT; + while let Ok(op2_tag) = Op2::try_from(i) { + let tag = Tag::Op2(op2_tag); + let tag_2 = Tag::pos(tag.index()).unwrap(); + assert_eq!(tag, tag_2); + i += 1; + } + } +} diff --git a/src/proof/tests/nova_tests_lem.rs b/src/proof/tests/nova_tests_lem.rs index b0ed617390..74499a580a 100644 --- a/src/proof/tests/nova_tests_lem.rs +++ b/src/proof/tests/nova_tests_lem.rs @@ -6,7 +6,7 @@ use crate::{ eval::lang::{Coproc, Lang}, lem::{ store::{intern_ptrs, Store}, - Tag, + tag::Tag, }, num::Num, proof::nova::C1LEM, diff --git a/src/tag.rs b/src/tag.rs index cb2a8c2686..dc2b0eb207 100644 --- a/src/tag.rs +++ b/src/tag.rs @@ -20,6 +20,7 @@ pub trait Tag: } } +pub(crate) const EXPR_TAG_INIT: u16 = 0b0000_0000_0000_0000; /// A tag for expressions. Note that ExprTag, ContTag, Op1, Op2 all live in the same u16 namespace #[derive( Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize_repr, Deserialize_repr, TryFromRepr, @@ -27,7 +28,7 @@ pub trait Tag: #[cfg_attr(not(target_arch = "wasm32"), derive(Arbitrary))] #[repr(u16)] pub enum ExprTag { - Nil = 0b0000_0000_0000_0000, + Nil = EXPR_TAG_INIT, Cons, Sym, Fun, @@ -89,6 +90,7 @@ impl Tag for ExprTag { } } +pub(crate) const CONT_TAG_INIT: u16 = 0b0001_0000_0000_0000; /// A tag for continuations. Note that ExprTag, ContTag, Op1, Op2 all live in the same u16 namespace #[derive( Serialize_repr, Deserialize_repr, Debug, Copy, Clone, PartialEq, Eq, Hash, TryFromRepr, @@ -96,7 +98,7 @@ impl Tag for ExprTag { #[cfg_attr(not(target_arch = "wasm32"), derive(Arbitrary))] #[repr(u16)] pub enum ContTag { - Outermost = 0b0001_0000_0000_0000, + Outermost = CONT_TAG_INIT, Call0, Call, Call2, @@ -168,6 +170,7 @@ impl fmt::Display for ContTag { } } +pub(crate) const OP1_TAG_INIT: u16 = 0b0010_0000_0000_0000; #[derive( Copy, Clone, @@ -183,7 +186,7 @@ impl fmt::Display for ContTag { #[cfg_attr(not(target_arch = "wasm32"), derive(Arbitrary))] #[repr(u16)] pub enum Op1 { - Car = 0b0010_0000_0000_0000, + Car = OP1_TAG_INIT, Cdr, Atom, Emit, @@ -297,6 +300,7 @@ impl fmt::Display for Op1 { } } +pub(crate) const OP2_TAG_INIT: u16 = 0b0011_0000_0000_0000; #[derive( Copy, Clone, @@ -312,7 +316,7 @@ impl fmt::Display for Op1 { #[cfg_attr(not(target_arch = "wasm32"), derive(Arbitrary))] #[repr(u16)] pub enum Op2 { - Sum = 0b0011_0000_0000_0000, + Sum = OP2_TAG_INIT, Diff, Product, Quotient, From 250b2ccd212412fc03c8db37c69d1ff8cc28fa67 Mon Sep 17 00:00:00 2001 From: Gabriel Barreto Date: Fri, 5 Jan 2024 14:12:46 -0300 Subject: [PATCH 12/12] Better conversion functions --- Cargo.toml | 1 + src/lem/tag.rs | 184 +++++++++++++------------------------------------ src/tag.rs | 29 +++++++- 3 files changed, 77 insertions(+), 137 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 94b549ddee..70786d95b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ serde = { workspace = true, features = ["derive"] } serde_bytes = "0.11.12" serde_json = { workspace = true } serde_repr = "0.1.14" +strum = { version = "0.25", features = ["derive"] } tap = "1.0.1" stable_deref_trait = "1.2.0" thiserror = { workspace = true } diff --git a/src/lem/tag.rs b/src/lem/tag.rs index 76c2e4f9ad..673e0d6d41 100644 --- a/src/lem/tag.rs +++ b/src/lem/tag.rs @@ -1,9 +1,13 @@ use anyhow::{bail, Result}; use serde::{Deserialize, Serialize}; +use strum::EnumCount; use crate::{ field::LurkField, - tag::{ContTag, ExprTag, Op1, Op2, Tag as TagTrait}, + tag::{ + ContTag, ExprTag, Op1, Op2, Tag as TagTrait, CONT_TAG_INIT, EXPR_TAG_INIT, OP1_TAG_INIT, + OP2_TAG_INIT, + }, }; /// The LEM `Tag` is a wrapper around other types that are used as tags @@ -66,127 +70,43 @@ impl Tag { } pub fn pos(i: usize) -> Option { - match i { - 0 => Some(Self::Expr(ExprTag::Nil)), - 1 => Some(Self::Expr(ExprTag::Cons)), - 2 => Some(Self::Expr(ExprTag::Sym)), - 3 => Some(Self::Expr(ExprTag::Fun)), - 4 => Some(Self::Expr(ExprTag::Num)), - 5 => Some(Self::Expr(ExprTag::Thunk)), - 6 => Some(Self::Expr(ExprTag::Str)), - 7 => Some(Self::Expr(ExprTag::Char)), - 8 => Some(Self::Expr(ExprTag::Comm)), - 9 => Some(Self::Expr(ExprTag::U64)), - 10 => Some(Self::Expr(ExprTag::Key)), - 11 => Some(Self::Expr(ExprTag::Cproc)), - 12 => Some(Self::Cont(ContTag::Outermost)), - 13 => Some(Self::Cont(ContTag::Call0)), - 14 => Some(Self::Cont(ContTag::Call)), - 15 => Some(Self::Cont(ContTag::Call2)), - 16 => Some(Self::Cont(ContTag::Tail)), - 17 => Some(Self::Cont(ContTag::Error)), - 18 => Some(Self::Cont(ContTag::Lookup)), - 19 => Some(Self::Cont(ContTag::Unop)), - 20 => Some(Self::Cont(ContTag::Binop)), - 21 => Some(Self::Cont(ContTag::Binop2)), - 22 => Some(Self::Cont(ContTag::If)), - 23 => Some(Self::Cont(ContTag::Let)), - 24 => Some(Self::Cont(ContTag::LetRec)), - 25 => Some(Self::Cont(ContTag::Dummy)), - 26 => Some(Self::Cont(ContTag::Terminal)), - 27 => Some(Self::Cont(ContTag::Emit)), - 28 => Some(Self::Cont(ContTag::Cproc)), - 29 => Some(Self::Op1(Op1::Car)), - 30 => Some(Self::Op1(Op1::Cdr)), - 31 => Some(Self::Op1(Op1::Atom)), - 32 => Some(Self::Op1(Op1::Emit)), - 33 => Some(Self::Op1(Op1::Open)), - 34 => Some(Self::Op1(Op1::Secret)), - 35 => Some(Self::Op1(Op1::Commit)), - 36 => Some(Self::Op1(Op1::Num)), - 37 => Some(Self::Op1(Op1::Comm)), - 38 => Some(Self::Op1(Op1::Char)), - 39 => Some(Self::Op1(Op1::Eval)), - 40 => Some(Self::Op1(Op1::U64)), - 41 => Some(Self::Op2(Op2::Sum)), - 42 => Some(Self::Op2(Op2::Diff)), - 43 => Some(Self::Op2(Op2::Product)), - 44 => Some(Self::Op2(Op2::Quotient)), - 45 => Some(Self::Op2(Op2::Equal)), - 46 => Some(Self::Op2(Op2::NumEqual)), - 47 => Some(Self::Op2(Op2::Less)), - 48 => Some(Self::Op2(Op2::Greater)), - 49 => Some(Self::Op2(Op2::LessEqual)), - 50 => Some(Self::Op2(Op2::GreaterEqual)), - 51 => Some(Self::Op2(Op2::Cons)), - 52 => Some(Self::Op2(Op2::StrCons)), - 53 => Some(Self::Op2(Op2::Begin)), - 54 => Some(Self::Op2(Op2::Hide)), - 55 => Some(Self::Op2(Op2::Modulo)), - 56 => Some(Self::Op2(Op2::Eval)), - _ => None, + let mut last = 0; + if (last..last + ExprTag::COUNT).contains(&i) { + let j = i + EXPR_TAG_INIT as usize - last; + let expr_tag = (j as u16).try_into().expect("unreachable"); + return Some(Tag::Expr(expr_tag)); } + last += ExprTag::COUNT; + if (last..last + ContTag::COUNT).contains(&i) { + let j = i + CONT_TAG_INIT as usize - last; + let cont_tag = (j as u16).try_into().expect("unreachable"); + return Some(Tag::Cont(cont_tag)); + } + last += ContTag::COUNT; + if (last..last + Op1::COUNT).contains(&i) { + let j = i + OP1_TAG_INIT as usize - last; + let op1_tag = (j as u16).try_into().expect("unreachable"); + return Some(Tag::Op1(op1_tag)); + } + last += Op1::COUNT; + if (last..last + Op2::COUNT).contains(&i) { + let j = i + OP2_TAG_INIT as usize - last; + let op2_tag = (j as u16).try_into().expect("unreachable"); + return Some(Tag::Op2(op2_tag)); + } + None } pub fn index(&self) -> usize { match self { - Self::Expr(ExprTag::Nil) => 0, - Self::Expr(ExprTag::Cons) => 1, - Self::Expr(ExprTag::Sym) => 2, - Self::Expr(ExprTag::Fun) => 3, - Self::Expr(ExprTag::Num) => 4, - Self::Expr(ExprTag::Thunk) => 5, - Self::Expr(ExprTag::Str) => 6, - Self::Expr(ExprTag::Char) => 7, - Self::Expr(ExprTag::Comm) => 8, - Self::Expr(ExprTag::U64) => 9, - Self::Expr(ExprTag::Key) => 10, - Self::Expr(ExprTag::Cproc) => 11, - Self::Cont(ContTag::Outermost) => 12, - Self::Cont(ContTag::Call0) => 13, - Self::Cont(ContTag::Call) => 14, - Self::Cont(ContTag::Call2) => 15, - Self::Cont(ContTag::Tail) => 16, - Self::Cont(ContTag::Error) => 17, - Self::Cont(ContTag::Lookup) => 18, - Self::Cont(ContTag::Unop) => 19, - Self::Cont(ContTag::Binop) => 20, - Self::Cont(ContTag::Binop2) => 21, - Self::Cont(ContTag::If) => 22, - Self::Cont(ContTag::Let) => 23, - Self::Cont(ContTag::LetRec) => 24, - Self::Cont(ContTag::Dummy) => 25, - Self::Cont(ContTag::Terminal) => 26, - Self::Cont(ContTag::Emit) => 27, - Self::Cont(ContTag::Cproc) => 28, - Self::Op1(Op1::Car) => 29, - Self::Op1(Op1::Cdr) => 30, - Self::Op1(Op1::Atom) => 31, - Self::Op1(Op1::Emit) => 32, - Self::Op1(Op1::Open) => 33, - Self::Op1(Op1::Secret) => 34, - Self::Op1(Op1::Commit) => 35, - Self::Op1(Op1::Num) => 36, - Self::Op1(Op1::Comm) => 37, - Self::Op1(Op1::Char) => 38, - Self::Op1(Op1::Eval) => 39, - Self::Op1(Op1::U64) => 40, - Self::Op2(Op2::Sum) => 41, - Self::Op2(Op2::Diff) => 42, - Self::Op2(Op2::Product) => 43, - Self::Op2(Op2::Quotient) => 44, - Self::Op2(Op2::Equal) => 45, - Self::Op2(Op2::NumEqual) => 46, - Self::Op2(Op2::Less) => 47, - Self::Op2(Op2::Greater) => 48, - Self::Op2(Op2::LessEqual) => 49, - Self::Op2(Op2::GreaterEqual) => 50, - Self::Op2(Op2::Cons) => 51, - Self::Op2(Op2::StrCons) => 52, - Self::Op2(Op2::Begin) => 53, - Self::Op2(Op2::Hide) => 54, - Self::Op2(Op2::Modulo) => 55, - Self::Op2(Op2::Eval) => 56, + Self::Expr(tag) => *tag as usize - EXPR_TAG_INIT as usize, + Self::Cont(tag) => *tag as usize - CONT_TAG_INIT as usize + ExprTag::COUNT, + Self::Op1(tag) => { + *tag as usize - OP1_TAG_INIT as usize + ExprTag::COUNT + ContTag::COUNT + } + Self::Op2(tag) => { + *tag as usize - OP2_TAG_INIT as usize + ExprTag::COUNT + ContTag::COUNT + Op1::COUNT + } } } } @@ -206,47 +126,41 @@ impl std::fmt::Display for Tag { #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::tag::{CONT_TAG_INIT, EXPR_TAG_INIT, OP1_TAG_INIT, OP2_TAG_INIT}; + use strum::IntoEnumIterator; #[test] fn pos_index_roundtrip() { - let mut i = 0; - while let Some(tag) = Tag::pos(i) { - let j = tag.index(); - assert_eq!(i, j); - i += 1; + for i in 0.. { + if let Some(tag) = Tag::pos(i) { + let j = tag.index(); + assert_eq!(i, j); + } else { + break; + } } - let mut i = EXPR_TAG_INIT; - while let Ok(expr_tag) = ExprTag::try_from(i) { + for expr_tag in ExprTag::iter() { let tag = Tag::Expr(expr_tag); let tag_2 = Tag::pos(tag.index()).unwrap(); assert_eq!(tag, tag_2); - i += 1; } - let mut i = CONT_TAG_INIT; - while let Ok(cont_tag) = ContTag::try_from(i) { + for cont_tag in ContTag::iter() { let tag = Tag::Cont(cont_tag); let tag_2 = Tag::pos(tag.index()).unwrap(); assert_eq!(tag, tag_2); - i += 1; } - let mut i = OP1_TAG_INIT; - while let Ok(op1_tag) = Op1::try_from(i) { + for op1_tag in Op1::iter() { let tag = Tag::Op1(op1_tag); let tag_2 = Tag::pos(tag.index()).unwrap(); assert_eq!(tag, tag_2); - i += 1; } - let mut i = OP2_TAG_INIT; - while let Ok(op2_tag) = Op2::try_from(i) { + for op2_tag in Op2::iter() { let tag = Tag::Op2(op2_tag); let tag_2 = Tag::pos(tag.index()).unwrap(); assert_eq!(tag, tag_2); - i += 1; } } } diff --git a/src/tag.rs b/src/tag.rs index dc2b0eb207..af2bc41673 100644 --- a/src/tag.rs +++ b/src/tag.rs @@ -3,6 +3,7 @@ use lurk_macros::TryFromRepr; use proptest_derive::Arbitrary; use serde_repr::{Deserialize_repr, Serialize_repr}; use std::{convert::TryFrom, fmt}; +use strum::{EnumCount, EnumIter}; use crate::field::LurkField; @@ -23,7 +24,17 @@ pub trait Tag: pub(crate) const EXPR_TAG_INIT: u16 = 0b0000_0000_0000_0000; /// A tag for expressions. Note that ExprTag, ContTag, Op1, Op2 all live in the same u16 namespace #[derive( - Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize_repr, Deserialize_repr, TryFromRepr, + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + Serialize_repr, + Deserialize_repr, + TryFromRepr, + EnumCount, + EnumIter, )] #[cfg_attr(not(target_arch = "wasm32"), derive(Arbitrary))] #[repr(u16)] @@ -93,7 +104,17 @@ impl Tag for ExprTag { pub(crate) const CONT_TAG_INIT: u16 = 0b0001_0000_0000_0000; /// A tag for continuations. Note that ExprTag, ContTag, Op1, Op2 all live in the same u16 namespace #[derive( - Serialize_repr, Deserialize_repr, Debug, Copy, Clone, PartialEq, Eq, Hash, TryFromRepr, + Serialize_repr, + Deserialize_repr, + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + TryFromRepr, + EnumCount, + EnumIter, )] #[cfg_attr(not(target_arch = "wasm32"), derive(Arbitrary))] #[repr(u16)] @@ -182,6 +203,8 @@ pub(crate) const OP1_TAG_INIT: u16 = 0b0010_0000_0000_0000; Serialize_repr, Deserialize_repr, TryFromRepr, + EnumCount, + EnumIter, )] #[cfg_attr(not(target_arch = "wasm32"), derive(Arbitrary))] #[repr(u16)] @@ -312,6 +335,8 @@ pub(crate) const OP2_TAG_INIT: u16 = 0b0011_0000_0000_0000; Serialize_repr, Deserialize_repr, TryFromRepr, + EnumCount, + EnumIter, )] #[cfg_attr(not(target_arch = "wasm32"), derive(Arbitrary))] #[repr(u16)]