Add data root field inside header extrinsic root (#74)

* add: data_root field inside extrinsic_root

* add: ext decoding and data root computation

* add: RPC call for querying data proof

* fix: RPC proof generation

* fix: missing code after cherry pick

* fix: subxt config

* rebase develop

* rebase develop

* add hash

Co-authored-by: Kailas <kailaskr05@gmail.com>

avail-subxt/src/lib.rs

@@ -225,6 +225,9 @@ pub struct KateCommitment {
 	pub rows: u16,
 	/// Cols
 	pub cols: u16,
+	/// The merkle root of the data submissions
+	#[serde(rename = "dataRoot")]
+	pub data_root: [u8; 32],
 }
 
 impl MallocSizeOf for KateCommitment {
@@ -233,6 +236,7 @@ impl MallocSizeOf for KateCommitment {
 			+ self.commitment.size_of(ops)
 			+ self.rows.size_of(ops)
 			+ self.cols.size_of(ops)
+			+ self.data_root.size_of(ops)
 	}
 }
 
@@ -301,6 +305,7 @@ impl Header for DaHeader {
 				commitment: vec![],
 				rows: 0,
 				cols: 0,
+				data_root: [0; 32],
 			},
 			digest,
 			app_data_lookup: DataLookup {

pallets/system/Cargo.toml

@@ -20,6 +20,7 @@ da-primitives = { path = "../../primitives", default-features = false }
 impl-trait-for-tuples = "0.2.1"
 static_assertions = "1.1.0"
 log = { version = "0.4.14", default-features = false }
+rs_merkle = { version = "1.2.0", default-features = false }
 
 # Substrate
 serde = { version = "1.0.126", optional = true, features = ["derive"] }

pallets/system/src/header_builder.rs

@@ -1,9 +1,10 @@
-use codec::{Decode, Encode};
+use codec::{Compact, Decode, Encode, Error as DecodeError, Input};
 use da_primitives::{asdr::AppExtrinsic, traits::ExtendedHeader};
 use frame_support::traits::Randomness;
 pub use kate::Seed;
+use rs_merkle::{algorithms::Sha256, Hasher, MerkleTree};
 use scale_info::TypeInfo;
-use sp_runtime::traits::Hash;
+use sp_runtime::{traits::Hash, AccountId32, MultiAddress, MultiSignature};
 use sp_runtime_interface::{pass_by::PassByCodec, runtime_interface};
 use sp_std::vec::Vec;
 
@@ -124,7 +125,27 @@ pub trait HostedHeaderBuilder {
 			(kate_commitment, block_dims, data_index)
 		};
 
-		let extrinsics = app_extrinsics.into_iter().map(|e| e.data).collect();
+		let extrinsics: Vec<Vec<u8>> = app_extrinsics.into_iter().map(|e| e.data).collect();
+		let avail_extrinsics = extrinsics
+			.iter()
+			.filter_map(|e| <AvailExtrinsic>::decode(&mut &e[..]).ok())
+			.collect::<Vec<_>>();
+
+		let data_root: [u8; 32] = if avail_extrinsics.len() > 0 {
+			log::debug!("Decoded some avail extrinsics.");
+			let leaves: Vec<[u8; 32]> = avail_extrinsics
+				.iter()
+				.map(|x| Sha256::hash(&x.data))
+				.collect();
+
+			let data_tree = MerkleTree::<Sha256>::from_leaves(&leaves);
+			data_tree.root().expect("Data Root computation failed")
+		} else {
+			Default::default()
+		};
+
+		log::debug!("Avail Data Root: {:?}\n", data_root);
+
 		let root_hash = da::Hasher::ordered_trie_root(extrinsics);
 
 		let storage_root = da::Hash::decode(&mut &sp_io::storage::root()[..])
@@ -146,6 +167,7 @@ pub trait HostedHeaderBuilder {
 			kate_commitment,
 			block_dims.rows as u16,
 			block_dims.cols as u16,
+			data_root,
 		);
 
 		<da::Header as ExtendedHeader>::new(
@@ -158,3 +180,86 @@ pub trait HostedHeaderBuilder {
 		)
 	}
 }
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct AvailExtrinsic {
+	pub app_id: u32,
+	pub signature: Option<MultiSignature>,
+	pub data: Vec<u8>,
+}
+
+pub type AvailSignedExtra = ((), (), (), AvailMortality, Nonce, (), Balance, u32);
+
+#[derive(Decode)]
+pub struct Balance(#[codec(compact)] u128);
+
+#[derive(Decode)]
+pub struct Nonce(#[codec(compact)] u32);
+
+pub enum AvailMortality {
+	Immortal,
+	Mortal(u64, u64),
+}
+
+impl Decode for AvailMortality {
+	fn decode<I: Input>(input: &mut I) -> Result<Self, DecodeError> {
+		let first = input.read_byte()?;
+		if first == 0 {
+			Ok(Self::Immortal)
+		} else {
+			let encoded = first as u64 + ((input.read_byte()? as u64) << 8);
+			let period = 2 << (encoded % (1 << 4));
+			let quantize_factor = (period >> 12).max(1);
+			let phase = (encoded >> 4) * quantize_factor;
+			if period >= 4 && phase < period {
+				Ok(Self::Mortal(period, phase))
+			} else {
+				Err("Invalid period and phase".into())
+			}
+		}
+	}
+}
+
+const EXTRINSIC_VERSION: u8 = 4;
+impl Decode for AvailExtrinsic {
+	fn decode<I: Input>(input: &mut I) -> Result<AvailExtrinsic, DecodeError> {
+		// This is a little more complicated than usual since the binary format must be compatible
+		// with substrate's generic `Vec<u8>` type. Basically this just means accepting that there
+		// will be a prefix of vector length (we don't need
+		// to use this).
+		let _length_do_not_remove_me_see_above: Compact<u32> = Decode::decode(input)?;
+
+		let version = input.read_byte()?;
+
+		let is_signed = version & 0b1000_0000 != 0;
+		let version = version & 0b0111_1111;
+		if version != EXTRINSIC_VERSION {
+			return Err("Invalid transaction version".into());
+		}
+		let (app_id, signature) = if is_signed {
+			let _address = <MultiAddress<AccountId32, u32>>::decode(input)?;
+			let sig = MultiSignature::decode(input)?;
+			let extra = <AvailSignedExtra>::decode(input)?;
+			let app_id = extra.7;
+
+			(app_id, Some(sig))
+		} else {
+			return Err("Not signed".into());
+		};
+
+		let section: u8 = Decode::decode(input)?;
+		let method: u8 = Decode::decode(input)?;
+
+		let data: Vec<u8> = match (section, method) {
+			// TODO: Define these pairs as enums or better yet - make a dependency on substrate enums if possible
+			(29, 1) => Decode::decode(input)?,
+			_ => return Err("Not Avail Extrinsic".into()),
+		};
+
+		Ok(Self {
+			app_id,
+			signature,
+			data,
+		})
+	}
+}

primitives/src/header.rs

@@ -303,6 +303,7 @@ where
 
 #[cfg(all(test, feature = "std"))]
 mod tests {
+
 	use super::*;
 
 	#[test]
@@ -346,4 +347,61 @@ mod tests {
 			u64::max_value() as u128 + 1
 		);
 	}
+
+	#[test]
+	fn ensure_format_is_unchanged() {
+		use sp_runtime::{
+			generic::{Digest, DigestItem},
+			traits::BlakeTwo256,
+		};
+
+		use crate::KateCommitment;
+		let extrinsic_root = KateCommitment {
+			hash: BlakeTwo256::hash(b"4"),
+			rows: 1,
+			cols: 4,
+			commitment: vec![
+				128, 233, 73, 235, 218, 245, 193, 62, 9, 100, 156, 88, 124, 107, 25, 5, 251, 119,
+				11, 74, 104, 67, 171, 170, 198, 180, 19, 227, 167, 64, 93, 152, 37, 172, 118, 77,
+				178, 52, 29, 185, 183, 150, 89, 101, 7, 62, 151, 89, 128, 233, 73, 235, 218, 245,
+				193, 62, 9, 100, 156, 88, 124, 107, 25, 5, 251, 119, 11, 74, 104, 67, 171, 170,
+				198, 180, 19, 227, 167, 64, 93, 152, 37, 172, 118, 77, 178, 52, 29, 185, 183, 150,
+				89, 101, 7, 62, 151, 89,
+			],
+			data_root: [
+				63, 191, 50, 39, 146, 108, 250, 63, 65, 103, 119, 30, 90, 217, 28, 250, 44, 45,
+				112, 144, 102, 124, 224, 30, 145, 28, 169, 11, 79, 49, 91, 17,
+			],
+		};
+		let data_lookup = DataLookup {
+			size: 1,
+			index: vec![],
+		};
+		let header = Header::<u32, BlakeTwo256> {
+			parent_hash: BlakeTwo256::hash(b"1"),
+			number: 2,
+			state_root: BlakeTwo256::hash(b"3"),
+			extrinsics_root: extrinsic_root,
+			digest: Digest {
+				logs: vec![DigestItem::Other(b"5".to_vec())],
+			},
+			app_data_lookup: data_lookup,
+		};
+		println!("{:?}", header);
+		let encoded = header.encode();
+		println!("{:?}", encoded);
+		assert_eq!(encoded, vec![
+			146, 205, 245, 120, 196, 112, 133, 165, 153, 34, 86, 240, 220, 249, 125, 11, 25, 241,
+			241, 201, 222, 77, 95, 227, 12, 58, 206, 97, 145, 182, 229, 219, 8, 88, 19, 72, 51,
+			123, 15, 62, 20, 134, 32, 23, 61, 170, 165, 249, 77, 0, 216, 129, 112, 93, 203, 240,
+			170, 131, 239, 218, 186, 97, 210, 237, 225, 235, 134, 73, 33, 73, 151, 87, 78, 32, 196,
+			100, 56, 138, 23, 36, 32, 210, 84, 3, 104, 43, 187, 184, 12, 73, 104, 49, 200, 204, 31,
+			143, 13, 129, 1, 128, 233, 73, 235, 218, 245, 193, 62, 9, 100, 156, 88, 124, 107, 25,
+			5, 251, 119, 11, 74, 104, 67, 171, 170, 198, 180, 19, 227, 167, 64, 93, 152, 37, 172,
+			118, 77, 178, 52, 29, 185, 183, 150, 89, 101, 7, 62, 151, 89, 128, 233, 73, 235, 218,
+			245, 193, 62, 9, 100, 156, 88, 124, 107, 25, 5, 251, 119, 11, 74, 104, 67, 171, 170,
+			198, 180, 19, 227, 167, 64, 93, 152, 37, 172, 118, 77, 178, 52, 29, 185, 183, 150, 89,
+			101, 7, 62, 151, 89, 1, 0, 4, 0, 4, 0, 4, 53, 1, 0, 0, 0, 0
+		],);
+	}
 }