feat: fallback preimage recovery w/o debug_dbGet (#30)

Cargo.toml

@@ -37,6 +37,7 @@ futures = "0.3"
 url = "2.3"
 thiserror = "1.0.61"
 hex-literal = "0.4.1"
+rayon = "1.10.0"
 
 # workspace
 rsp-rpc-db = { path = "./crates/storage/rpc-db" }

README.md

@@ -22,9 +22,9 @@ and the command `rsp` will be installed.
 
 ### RPC Node Requirement
 
-RSP fetches block and state data from a JSON-RPC node. **But, you must use a RPC node that supports the `debug_dbGet` endpoint.**
+RSP fetches block and state data from a JSON-RPC node. It's recommended that you use a RPC node that supports the `debug_dbGet` endpoint.
 
-This is required because in some cases the host needs to recover the preimage of a [Merkle Patricia Trie](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/) node that's referenced by hash. To do this, the host utilizes the [`debug_dbGet` endpoint](https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-debug#debugdbget) of a Geth node running with options `--state.scheme=hash`, which is the default, and `--gcmode=archive`. An example command for running the node is:
+This is recommended because in some cases the host needs to recover the preimage of a [Merkle Patricia Trie](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/) node that's referenced by hash. To do this, the host utilizes the [`debug_dbGet` endpoint](https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-debug#debugdbget) of a Geth node running with options `--state.scheme=hash`, which is the default, and `--gcmode=archive`. An example command for running the node is:
 
 ```bash
 geth \
@@ -33,7 +33,7 @@ geth \
   --http.api=eth,debug
 ```
 
-When running the host CLI or integration tests, **make sure to use an RPC URL pointing to a Geth node running with said options**, or errors will arise when preimage recovery is needed. You can reach out to the Succinct team to access an RPC URL that supports this endpoint.
+However, in the absence of the `debug_dbGet` method, the host is able to fall back to a less efficient process of recovering the preimages via the standard `eth_getProof`. The fallback works in most cases but not all, so if you encounter a preimage recovery failure, you can reach out to the Succinct team to access an RPC URL that supports `debug_dbGet`.
 
 > [!TIP]
 >

crates/executor/host/Cargo.toml

@@ -22,7 +22,7 @@ tracing.workspace = true
 rsp-rpc-db.workspace = true
 rsp-witness-db.workspace = true
 rsp-client-executor.workspace = true
-rsp-mpt.workspace = true
+rsp-mpt = { workspace = true, features = ["preimage_context"] }
 rsp-primitives.workspace = true
 
 # reth

crates/mpt/Cargo.toml

@@ -34,3 +34,9 @@ alloy-rpc-types.workspace = true
 alloy-trie.workspace = true
 hex-literal.workspace = true
 tracing-subscriber = "0.3.18"
+
+rsp-mpt = { path = ".", features = ["preimage_context"] }
+
+[features]
+default = []
+preimage_context = []

crates/mpt/src/lib.rs

@@ -13,6 +13,18 @@ use reth_trie::{
 use revm_primitives::{keccak256, HashMap};
 use rsp_primitives::storage::ExtDatabaseRef;
 
+#[cfg(feature = "preimage_context")]
+use rsp_primitives::storage::PreimageContext;
+
+/// Additional context for preimage recovery when calculating trie root. `Some` when calculating
+/// storage trie root and `None` when calculating state trie root.
+#[cfg(feature = "preimage_context")]
+type RootContext = Option<Address>;
+
+/// No additional context is needed since the `preimage_context` feature is disabled.
+#[cfg(not(feature = "preimage_context"))]
+type RootContext = ();
+
 /// Computes the state root of a block's Merkle Patricia Trie given an [ExecutionOutcome] and a list
 /// of [EIP1186AccountProofResponse] storage proofs.
 pub fn compute_state_root<DB>(
@@ -57,6 +69,11 @@ where
         let root = if proof.storage_proofs.is_empty() {
             proof.storage_root
         } else {
+            #[cfg(feature = "preimage_context")]
+            let context = Some(address);
+            #[cfg(not(feature = "preimage_context"))]
+            let context = ();
+
             compute_root_from_proofs(
                 storage_prefix_sets.freeze().iter().map(|storage_nibbles| {
                     let hashed_slot = B256::from_slice(&storage_nibbles.pack());
@@ -75,11 +92,17 @@ where
                     (storage_nibbles.clone(), encoded, storage_proof.proof.clone())
                 }),
                 db,
+                context,
             )?
         };
         storage_roots.insert(hashed_address, root);
     }
 
+    #[cfg(feature = "preimage_context")]
+    let context = None;
+    #[cfg(not(feature = "preimage_context"))]
+    let context = ();
+
     // Compute the state root of the entire trie.
     let mut rlp_buf = Vec::with_capacity(128);
     compute_root_from_proofs(
@@ -101,13 +124,15 @@ where
             (account_nibbles.clone(), encoded, proof.proof.clone())
         }),
         db,
+        context,
     )
 }
 
 /// Given a list of Merkle-Patricia proofs, compute the root of the trie.
 fn compute_root_from_proofs<DB>(
     items: impl IntoIterator<Item = (Nibbles, Option<Vec<u8>>, Vec<Bytes>)>,
     db: &DB,
+    #[allow(unused)] root_context: RootContext,
 ) -> eyre::Result<B256>
 where
     DB: ExtDatabaseRef<Error: std::fmt::Debug>,
@@ -277,7 +302,16 @@ where
                     // technically have to modify this branch node, but the `alloy-trie` hash
                     // builder handles this automatically when supplying child nodes.
 
+                    #[cfg(feature = "preimage_context")]
+                    let preimage = db
+                        .trie_node_ref_with_context(
+                            branch_hash,
+                            PreimageContext { address: &root_context, branch_path: &path },
+                        )
+                        .unwrap();
+                    #[cfg(not(feature = "preimage_context"))]
                     let preimage = db.trie_node_ref(branch_hash).unwrap();
+
                     match TrieNode::decode(&mut &preimage[..]).unwrap() {
                         TrieNode::Branch(_) => {
                             // This node is a branch node that's referenced by hash. There's no need
@@ -373,6 +407,14 @@ mod tests {
 
             panic!("missing preimage for test")
         }
+
+        fn trie_node_ref_with_context(
+            &self,
+            hash: B256,
+            _context: PreimageContext<'_>,
+        ) -> Result<Bytes, Self::Error> {
+            self.trie_node_ref(hash)
+        }
     }
 
     #[test]
@@ -468,6 +510,7 @@ cb10a951f0e82cf2e461b98c4e5afb0348ccab5bb42180808080808080808080808080"
                 ],
             )],
             &TestTrieDb::new(),
+            None,
         )
         .unwrap();
 
@@ -586,6 +629,7 @@ f2e461b98c4e5afb0348ccab5bb421808080808080808080808080"
                 ),
             ],
             &TestTrieDb::new(),
+            None,
         )
         .unwrap();
 
@@ -652,6 +696,7 @@ f2e461b98c4e5afb0348ccab5bb421808080808080808080808080"
                 ),
             ],
             &TestTrieDb::new(),
+            None,
         )
         .unwrap();
 
@@ -712,6 +757,7 @@ f2e461b98c4e5afb0348ccab5bb421808080808080808080808080"
                 ),
             ],
             &TestTrieDb::new(),
+            None,
         )
         .unwrap();
 
@@ -756,6 +802,7 @@ f2e461b98c4e5afb0348ccab5bb421808080808080808080808080"
                 ],
             )],
             &TestTrieDb::new(),
+            None,
         )
         .unwrap();
 

crates/primitives/src/storage.rs

@@ -1,4 +1,5 @@
-use reth_primitives::{Bytes, B256};
+use reth_primitives::{Address, Bytes, B256};
+use reth_trie::Nibbles;
 
 /// Custom database access methods implemented by RSP storage backends.
 pub trait ExtDatabaseRef {
@@ -7,4 +8,22 @@ pub trait ExtDatabaseRef {
 
     /// Gets the preimage of a trie node given its Keccak hash.
     fn trie_node_ref(&self, hash: B256) -> Result<Bytes, Self::Error>;
+
+    /// Gets the preimage of a trie node given its Keccak hash, with additional context that could
+    /// be helpful when the program is not running in a constrained environment.
+    fn trie_node_ref_with_context(
+        &self,
+        hash: B256,
+        context: PreimageContext,
+    ) -> Result<Bytes, Self::Error>;
+}
+
+/// Additional context for retrieving trie node preimages. These are useful when the JSON-RPC node
+/// does not serve the `debug_dbGet`.
+pub struct PreimageContext<'a> {
+    /// The account address if calculating a storage trie root; `None` if calculating the state
+    /// root.
+    pub address: &'a Option<Address>,
+    /// The trie key path of the branch child containing the hash whose preimage is being fetched.
+    pub branch_path: &'a Nibbles,
 }

crates/storage/rpc-db/Cargo.toml

@@ -12,6 +12,7 @@ tokio.workspace = true
 futures.workspace = true
 thiserror.workspace = true
 tracing.workspace = true
+rayon.workspace = true
 
 # workspace
 rsp-witness-db.workspace = true
@@ -21,6 +22,7 @@ rsp-primitives.workspace = true
 reth-primitives.workspace = true
 reth-storage-errors.workspace = true
 reth-revm.workspace = true
+reth-trie.workspace = true
 
 # revm
 revm-primitives.workspace = true