update: add merkle root to tendermint bundle summary (#137)

* update: add merkle root to tendermint bundle summary

* update: add merkle.ts to sdk

* chore: kyvejs/sdk

* fix: lint errors

* chore: added merkle root summary to tendermint-bsync

---------

Co-authored-by: Troy Kessler <troy.kessler99@gmail.com>

common/sdk/src/clients/lcd-client/client.ts

@@ -1,4 +1,3 @@
-import { V1BundlesLCDClient } from "./v1/bundles/query";
 import { BundlesModuleLCDClient } from "./bundles/v1beta1/query";
 import { DelegationModuleLCDClient } from "./delegation/v1beta1/query";
 import { FundersModuleLCDClient } from "./funders/v1beta1/query";
@@ -7,6 +6,7 @@ import { PoolModuleLCDClient } from "./pool/v1beta1/query";
 import { QueryModuleLCDClient } from "./query/v1beta1/query";
 import { StakersModuleLCDClient } from "./stakers/v1beta1/query";
 import { TeamModuleLCDClient } from "./team/v1beta1/query";
+import { V1BundlesLCDClient } from "./v1/bundles/query";
 class KyveLCDClient {
   public v1: {
     bundles: V1BundlesLCDClient;

common/sdk/src/index.ts

@@ -2,5 +2,6 @@ export * from "./clients/lcd-client/client";
 export { default as KyveClient } from "./clients/rpc-client/client";
 export { default as KyveWebClient } from "./clients/rpc-client/web.client";
 export * as constants from "./constants";
+export { dataItemToSha256, generateMerkleRoot } from "./merkle/merkle";
 export { registry } from "./registry/tx.registry";
 export { KyveSDK as default } from "./sdk";

common/sdk/src/merkle/merkle.ts

@@ -0,0 +1,41 @@
+import * as crypto from "@cosmjs/crypto";
+
+export function createHashesFromBundle(bundle: any[]): Uint8Array[] {
+  return bundle.map((dataItem) => dataItemToSha256(dataItem));
+}
+
+export function dataItemToSha256(data: any): Uint8Array {
+  // Encode the serialized object to UTF-8
+  const encoded_obj: Uint8Array = Buffer.from(JSON.stringify(data), "utf-8");
+  // Calculate the SHA-256 hash
+  return crypto.sha256(encoded_obj);
+}
+
+export function generateMerkleRoot(hashes: Uint8Array[]): Uint8Array {
+  if (!hashes || hashes.length == 0) {
+    return Buffer.from("");
+  }
+
+  // Ensure number of hashes (leafs) are even by copying the
+  // last hash (the very right leaf) if the amount is odd
+  if (hashes.length % 2 !== 0) {
+    hashes.push(hashes[hashes.length - 1]);
+  }
+
+  const combinedHashes: Uint8Array[] = [];
+  for (let i = 0; i < hashes.length; i += 2) {
+    const hashesConcatenated = new Uint8Array([
+      ...Array.from(hashes[i]),
+      ...Array.from(hashes[i + 1]),
+    ]);
+    const hash = crypto.sha256(hashesConcatenated);
+    combinedHashes.push(hash);
+  }
+
+  // If the combinedHashes length is 1, it means that we have the merkle root already,
+  // and we can return the hex representation
+  if (combinedHashes.length === 1) {
+    return combinedHashes[0];
+  }
+  return generateMerkleRoot(combinedHashes);
+}

integrations/tendermint-bsync/package.json

@@ -23,6 +23,7 @@
   },
   "dependencies": {
     "@kyvejs/protocol": "1.1.7",
+    "@kyvejs/sdk": "1.2.0",
     "axios": "^0.27.2"
   },
   "devDependencies": {

integrations/tendermint-bsync/src/runtime.ts

@@ -1,6 +1,8 @@
 import { DataItem, IRuntime, Validator, VOTE } from '@kyvejs/protocol';
 import { name, version } from '../package.json';
 import axios from 'axios';
+import { createHashesFromTendermintBundle } from './utils/merkle';
+import { generateMerkleRoot } from '@kyvejs/sdk';
 
 // TendermintBSync config
 interface IConfig {
@@ -82,8 +84,12 @@ export default class TendermintBSync implements IRuntime {
   }
 
   async summarizeDataBundle(_: Validator, bundle: DataItem[]): Promise<string> {
-    // use latest block height as bundle summary
-    return bundle.at(-1)?.value?.header?.height ?? '';
+    const hashes: Uint8Array[] = createHashesFromTendermintBundle(bundle);
+    const merkleRoot: Uint8Array = generateMerkleRoot(hashes);
+
+    return JSON.stringify({
+      merkle_root: Buffer.from(merkleRoot).toString('hex'),
+    });
   }
 
   async nextKey(_: Validator, key: string): Promise<string> {

integrations/tendermint-bsync/src/utils/merkle.ts

@@ -0,0 +1,30 @@
+import * as crypto from '@cosmjs/crypto';
+import { dataItemToSha256, generateMerkleRoot } from '@kyvejs/sdk';
+
+// Creates an Array of hashes from an array of data items (bundle).
+// The hash of a data item consists of the Merkle root from the block and the block
+// results (only two leafs) and the key of the data item. This allows the
+// Trustless API to serve block and block results independently.
+export function createHashesFromTendermintBundle(bundle: any[]): Uint8Array[] {
+  return bundle.map((dataItem) => {
+    const blockHashes: Uint8Array[] = [
+      dataItemToSha256(dataItem.value?.block),
+      dataItemToSha256(dataItem.value?.block_results),
+    ];
+
+    const merkleRoot: Uint8Array = generateMerkleRoot(blockHashes);
+
+    return tendermintDataItemToSha256(dataItem.key, merkleRoot);
+  });
+}
+
+function tendermintDataItemToSha256(
+  key: string,
+  merkleRoot: Uint8Array
+): Uint8Array {
+  const keyBytes = crypto.sha256(Buffer.from(key, 'utf-8'));
+
+  const combined = Buffer.concat([keyBytes, merkleRoot]);
+
+  return crypto.sha256(combined);
+}

integrations/tendermint/package.json

@@ -23,6 +23,7 @@
   },
   "dependencies": {
     "@kyvejs/protocol": "1.1.7",
+    "@kyvejs/sdk": "1.2.0",
     "ajv": "^8.12.0",
     "axios": "^0.27.2",
     "dotenv": "^16.3.1"

integrations/tendermint/src/runtime.ts

@@ -4,6 +4,8 @@ import axios from 'axios';
 import Ajv from 'ajv';
 import block_schema from './schemas/block.json';
 import block_results_schema from './schemas/block_result.json';
+import { createHashesFromTendermintBundle } from './utils/merkle';
+import { generateMerkleRoot } from '@kyvejs/sdk';
 
 const ajv = new Ajv();
 
@@ -236,8 +238,12 @@ export default class Tendermint implements IRuntime {
   }
 
   async summarizeDataBundle(_: Validator, bundle: DataItem[]): Promise<string> {
-    // use latest block height as bundle summary
-    return bundle.at(-1)?.value?.block?.block?.header?.height ?? '';
+    const hashes: Uint8Array[] = createHashesFromTendermintBundle(bundle);
+    const merkleRoot: Uint8Array = generateMerkleRoot(hashes);
+
+    return JSON.stringify({
+      merkle_root: Buffer.from(merkleRoot).toString('hex'),
+    });
   }
 
   async nextKey(_: Validator, key: string): Promise<string> {

integrations/tendermint/src/utils/merkle.ts

@@ -0,0 +1,30 @@
+import * as crypto from '@cosmjs/crypto';
+import { dataItemToSha256, generateMerkleRoot } from '@kyvejs/sdk';
+
+// Creates an Array of hashes from an array of data items (bundle).
+// The hash of a data item consists of the Merkle root from the block and the block
+// results (only two leafs) and the key of the data item. This allows the
+// Trustless API to serve block and block results independently.
+export function createHashesFromTendermintBundle(bundle: any[]): Uint8Array[] {
+  return bundle.map((dataItem) => {
+    const blockHashes: Uint8Array[] = [
+      dataItemToSha256(dataItem.value?.block),
+      dataItemToSha256(dataItem.value?.block_results),
+    ];
+
+    const merkleRoot: Uint8Array = generateMerkleRoot(blockHashes);
+
+    return tendermintDataItemToSha256(dataItem.key, merkleRoot);
+  });
+}
+
+function tendermintDataItemToSha256(
+  key: string,
+  merkleRoot: Uint8Array
+): Uint8Array {
+  const keyBytes = crypto.sha256(Buffer.from(key, 'utf-8'));
+
+  const combined = Buffer.concat([keyBytes, merkleRoot]);
+
+  return crypto.sha256(combined);
+}

tools/kysor/src/commands/init.ts

@@ -4,7 +4,7 @@ import { Command } from "commander";
 import fs from "fs";
 import path from "path";
 
-import { USER_HOME, KYSOR_DIR } from "../utils/constants";
+import { KYSOR_DIR, USER_HOME } from "../utils/constants";
 
 const init = new Command("init").description("Init KYSOR");
 

tools/kysor/src/commands/valaccounts.ts

@@ -7,7 +7,7 @@ import path from "path";
 import prompts from "prompts";
 
 import { IValaccountConfig } from "../types/interfaces";
-import { FILE_ACCESS, USER_HOME, KYSOR_DIR } from "../utils/constants";
+import { FILE_ACCESS, KYSOR_DIR, USER_HOME } from "../utils/constants";
 
 const valaccounts = new Command("valaccounts").description(
   "Create and delete valaccounts"

tools/kysor/src/kysor.ts

@@ -9,7 +9,7 @@ import path from "path";
 
 import { IConfig, IValaccountConfig } from "./types/interfaces";
 import { getChecksum, setupLogger, startNodeProcess } from "./utils";
-import { ARCH, USER_HOME, KYSOR_DIR, PLATFORM } from "./utils/constants";
+import { ARCH, KYSOR_DIR, PLATFORM, USER_HOME } from "./utils/constants";
 
 const INFINITY_LOOP = true;