Multisig and Aggregation ISM Metadata Encoding (#3701)

### Description

- Implement multisig metadata encoding/decoding
- Implement aggregation metadata encoding/decoding
- Generates test metadata fixtures from solidity unit tests
- Test encoders using fixtures

### Drive By

- Make CI tests run topologically

### Related issues

- Fixes #3449
- Fixes #3451

### Backward compatibility

Yes

### Testing

Unit Tests

.changeset/green-ads-live.md

@@ -1,5 +1,5 @@
 ---
-"@hyperlane-xyz/cli": minor
+'@hyperlane-xyz/cli': minor
 ---
 
 Default to home directory for local registry

.changeset/sour-bats-sort.md

@@ -0,0 +1,6 @@
+---
+'@hyperlane-xyz/utils': minor
+'@hyperlane-xyz/sdk': minor
+---
+
+Implement aggregation and multisig ISM metadata encoding

package.json

@@ -25,7 +25,7 @@
     "prettier": "yarn workspaces foreach --since --parallel run prettier",
     "lint": "yarn workspaces foreach --all --parallel run lint",
     "test": "yarn workspaces foreach --all --parallel run test",
-    "test:ci": "yarn workspaces foreach --all --parallel run test:ci",
+    "test:ci": "yarn workspaces foreach --all --topological run test:ci",
     "coverage": "yarn workspaces foreach --all --parallel run coverage",
     "version:prepare": "yarn changeset version && yarn workspaces foreach --all --parallel run version:update && yarn install --no-immutable",
     "version:check": "yarn changeset status",

solidity/.gitignore

@@ -14,3 +14,4 @@ forge-cache
 docs
 flattened/
 buildArtifact.json
+fixtures/

solidity/README.md

@@ -6,14 +6,30 @@ Hyperlane Core contains the contracts and typechain artifacts for the Hyperlane
 
 ```bash
 # Install with NPM
-npm install @hyperlane-xyz/utils
+npm install @hyperlane-xyz/core
 
 # Or with Yarn
-yarn add @hyperlane-xyz/utils
+yarn add @hyperlane-xyz/core
 ```
 
 Note, this package uses [ESM Modules](https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c#pure-esm-package)
 
+## Build
+
+```bash
+yarn build
+```
+
+## Test
+
+```bash
+yarn test
+```
+
+### Fixtures
+
+Some forge tests may generate fixtures in the [fixtures](./fixtures/) directory. This allows [SDK](../typescript/sdk) tests to leverage forge fuzzing. These are git ignored and should not be committed.
+
 ## License
 
 Apache 2.0

solidity/foundry.toml

@@ -10,7 +10,10 @@ solc_version = '0.8.22'
 evm_version= 'paris'
 optimizer = true
 optimizer_runs = 999_999
-fs_permissions = [{ access = "read-write", path = "./"}]
+fs_permissions = [
+    { access = "read", path = "./script/avs/"},
+    { access = "write", path = "./fixtures" }
+]
 ignored_warnings_from = ['fx-portal']
 
 [profile.ci]

solidity/package.json

@@ -61,14 +61,14 @@
   "scripts": {
     "build": "yarn hardhat-esm compile && tsc && ./exportBuildArtifact.sh",
     "lint": "solhint contracts/**/*.sol",
-    "clean": "yarn hardhat-esm clean && rm -rf ./dist ./cache ./types ./coverage ./out ./forge-cache",
+    "clean": "yarn hardhat-esm clean && rm -rf ./dist ./cache ./types ./coverage ./out ./forge-cache ./fixtures",
     "coverage": "./coverage.sh",
     "docs": "forge doc",
     "hardhat-esm": "NODE_OPTIONS='--experimental-loader ts-node/esm/transpile-only --no-warnings=ExperimentalWarning' hardhat --config hardhat.config.cts",
     "prettier": "prettier --write ./contracts ./test",
     "test": "yarn hardhat-esm test && yarn test:forge",
     "test:hardhat": "yarn hardhat-esm test",
-    "test:forge": "forge test -vvv",
+    "test:forge": "mkdir -p ./fixtures/aggregation ./fixtures/multisig && forge test -vvv",
     "test:ci": "yarn test:hardhat && yarn test:forge --no-match-test testFork",
     "gas": "forge snapshot",
     "gas-ci": "yarn gas --check --tolerance 2 || (echo 'Manually update gas snapshot' && exit 1)",

solidity/test/isms/AggregationIsm.t.sol

@@ -3,19 +3,44 @@ pragma solidity ^0.8.13;
 
 import "forge-std/Test.sol";
 
+import "@openzeppelin/contracts/utils/Strings.sol";
+
 import {IAggregationIsm} from "../../contracts/interfaces/isms/IAggregationIsm.sol";
 import {StaticAggregationIsmFactory} from "../../contracts/isms/aggregation/StaticAggregationIsmFactory.sol";
 import {AggregationIsmMetadata} from "../../contracts/isms/libs/AggregationIsmMetadata.sol";
 import {TestIsm, ThresholdTestUtils} from "./IsmTestUtils.sol";
 
 contract AggregationIsmTest is Test {
+    using Strings for uint256;
+    using Strings for uint8;
+
+    string constant fixtureKey = "fixture";
+
     StaticAggregationIsmFactory factory;
     IAggregationIsm ism;
 
     function setUp() public {
         factory = new StaticAggregationIsmFactory();
     }
 
+    function fixtureAppendMetadata(
+        uint256 index,
+        bytes memory metadata
+    ) internal {
+        vm.serializeBytes(fixtureKey, index.toString(), metadata);
+    }
+
+    function fixtureAppendNull(uint256 index) internal {
+        vm.serializeString(fixtureKey, index.toString(), "null");
+    }
+
+    function writeFixture(bytes memory metadata, uint8 m) internal {
+        string memory path = string(
+            abi.encodePacked("./fixtures/aggregation/", m.toString(), ".json")
+        );
+        vm.writeJson(vm.serializeBytes(fixtureKey, "encoded", metadata), path);
+    }
+
     function deployIsms(
         uint8 m,
         uint8 n,
@@ -32,34 +57,37 @@ contract AggregationIsmTest is Test {
         return isms;
     }
 
-    function getMetadata(
-        uint8 m,
-        bytes32 seed
-    ) private view returns (bytes memory) {
+    function getMetadata(uint8 m, bytes32 seed) private returns (bytes memory) {
         (address[] memory choices, ) = ism.modulesAndThreshold("");
         address[] memory chosen = ThresholdTestUtils.choose(m, choices, seed);
         bytes memory offsets;
         uint32 start = 8 * uint32(choices.length);
         bytes memory metametadata;
+
         for (uint256 i = 0; i < choices.length; i++) {
             bool included = false;
             for (uint256 j = 0; j < chosen.length; j++) {
                 included = included || choices[i] == chosen[j];
             }
             if (included) {
-                bytes memory requiredMetadata = TestIsm(choices[i])
-                    .requiredMetadata();
-                uint32 end = start + uint32(requiredMetadata.length);
+                bytes memory metadata = TestIsm(choices[i]).requiredMetadata();
+                uint32 end = start + uint32(metadata.length);
                 uint64 offset = (uint64(start) << 32) | uint64(end);
                 offsets = bytes.concat(offsets, abi.encodePacked(offset));
                 start = end;
-                metametadata = abi.encodePacked(metametadata, requiredMetadata);
+                metametadata = abi.encodePacked(metametadata, metadata);
+                fixtureAppendMetadata(i, metadata);
             } else {
-                uint64 offset = 0;
-                offsets = bytes.concat(offsets, abi.encodePacked(offset));
+                offsets = bytes.concat(offsets, abi.encodePacked(uint64(0)));
+                fixtureAppendNull(i);
             }
         }
-        return abi.encodePacked(offsets, metametadata);
+
+        bytes memory encoded = abi.encodePacked(offsets, metametadata);
+
+        writeFixture(encoded, m);
+
+        return encoded;
     }
 
     function testVerify(uint8 m, uint8 n, bytes32 seed) public {

solidity/test/isms/MultisigIsm.t.sol

@@ -3,6 +3,8 @@ pragma solidity ^0.8.13;
 
 import "forge-std/Test.sol";
 
+import "@openzeppelin/contracts/utils/Strings.sol";
+
 import {IMultisigIsm} from "../../contracts/interfaces/isms/IMultisigIsm.sol";
 import {TestMailbox} from "../../contracts/test/TestMailbox.sol";
 import {StaticMerkleRootMultisigIsmFactory, StaticMessageIdMultisigIsmFactory} from "../../contracts/isms/multisig/StaticMultisigIsm.sol";
@@ -20,6 +22,13 @@ import {ThresholdTestUtils} from "./IsmTestUtils.sol";
 abstract contract AbstractMultisigIsmTest is Test {
     using Message for bytes;
     using TypeCasts for address;
+    using Strings for uint256;
+    using Strings for uint8;
+
+    string constant fixtureKey = "fixture";
+    string constant signatureKey = "signature";
+    string constant signaturesKey = "signatures";
+    string constant prefixKey = "prefix";
 
     uint32 constant ORIGIN = 11;
     StaticThresholdAddressSetFactory factory;
@@ -30,32 +39,88 @@ abstract contract AbstractMultisigIsmTest is Test {
 
     function metadataPrefix(
         bytes memory message
-    ) internal view virtual returns (bytes memory);
+    ) internal virtual returns (bytes memory);
+
+    function fixtureInit() internal {
+        vm.serializeUint(fixtureKey, "type", uint256(ism.moduleType()));
+        string memory prefix = vm.serializeString(prefixKey, "dummy", "dummy");
+        vm.serializeString(fixtureKey, "prefix", prefix);
+    }
+
+    function fixtureAppendSignature(
+        uint256 index,
+        uint8 v,
+        bytes32 r,
+        bytes32 s
+    ) internal {
+        vm.serializeUint(signatureKey, "v", uint256(v));
+        vm.serializeBytes32(signatureKey, "r", r);
+        string memory signature = vm.serializeBytes32(signatureKey, "s", s);
+        vm.serializeString(signaturesKey, index.toString(), signature);
+    }
+
+    function writeFixture(bytes memory metadata, uint8 m, uint8 n) internal {
+        vm.serializeString(
+            fixtureKey,
+            "signatures",
+            vm.serializeString(signaturesKey, "dummy", "dummy")
+        );
+
+        string memory fixturePath = string(
+            abi.encodePacked(
+                "./fixtures/multisig/",
+                m.toString(),
+                "-",
+                n.toString(),
+                ".json"
+            )
+        );
+        vm.writeJson(
+            vm.serializeBytes(fixtureKey, "encoded", metadata),
+            fixturePath
+        );
+    }
 
     function getMetadata(
         uint8 m,
         uint8 n,
         bytes32 seed,
         bytes memory message
     ) internal returns (bytes memory) {
-        uint32 domain = mailbox.localDomain();
-        uint256[] memory keys = addValidators(m, n, seed);
-        uint256[] memory signers = ThresholdTestUtils.choose(m, keys, seed);
+        bytes32 digest;
+        {
+            uint32 domain = mailbox.localDomain();
+            (bytes32 root, uint32 index) = merkleTreeHook.latestCheckpoint();
+            bytes32 messageId = message.id();
+            bytes32 merkleTreeAddress = address(merkleTreeHook)
+                .addressToBytes32();
+            digest = CheckpointLib.digest(
+                domain,
+                merkleTreeAddress,
+                root,
+                index,
+                messageId
+            );
+        }
 
-        (bytes32 root, uint32 index) = merkleTreeHook.latestCheckpoint();
-        bytes32 messageId = message.id();
-        bytes32 digest = CheckpointLib.digest(
-            domain,
-            address(merkleTreeHook).addressToBytes32(),
-            root,
-            index,
-            messageId
+        uint256[] memory signers = ThresholdTestUtils.choose(
+            m,
+            addValidators(m, n, seed),
+            seed
         );
+
         bytes memory metadata = metadataPrefix(message);
+        fixtureInit();
+
         for (uint256 i = 0; i < m; i++) {
             (uint8 v, bytes32 r, bytes32 s) = vm.sign(signers[i], digest);
             metadata = abi.encodePacked(metadata, r, s, v);
+
+            fixtureAppendSignature(i, v, r, s);
         }
+
+        writeFixture(metadata, m, n);
+
         return metadata;
     }
 
@@ -125,6 +190,9 @@ abstract contract AbstractMultisigIsmTest is Test {
 contract MerkleRootMultisigIsmTest is AbstractMultisigIsmTest {
     using TypeCasts for address;
     using Message for bytes;
+    using Strings for uint256;
+
+    string constant proofKey = "proof";
 
     function setUp() public {
         mailbox = new TestMailbox(ORIGIN);
@@ -135,17 +203,45 @@ contract MerkleRootMultisigIsmTest is AbstractMultisigIsmTest {
         mailbox.setRequiredHook(address(noopHook));
     }
 
+    function fixturePrefix(
+        uint32 checkpointIndex,
+        bytes32 merkleTreeAddress,
+        bytes32 messageId,
+        bytes32[32] memory proof
+    ) internal {
+        vm.serializeUint(prefixKey, "index", uint256(checkpointIndex));
+        vm.serializeBytes32(prefixKey, "merkleTree", merkleTreeAddress);
+        vm.serializeUint(prefixKey, "signedIndex", uint256(checkpointIndex));
+        vm.serializeBytes32(prefixKey, "id", messageId);
+
+        for (uint256 i = 0; i < 32; i++) {
+            vm.serializeBytes32(proofKey, i.toString(), proof[i]);
+        }
+        string memory proofString = vm.serializeString(
+            proofKey,
+            "dummy",
+            "dummy"
+        );
+        vm.serializeString(prefixKey, "proof", proofString);
+    }
+
     // TODO: test merkleIndex != signedIndex
     function metadataPrefix(
         bytes memory message
-    ) internal view override returns (bytes memory) {
+    ) internal override returns (bytes memory) {
         uint32 checkpointIndex = uint32(merkleTreeHook.count() - 1);
+        bytes32[32] memory proof = merkleTreeHook.proof();
+        bytes32 messageId = message.id();
+        bytes32 merkleTreeAddress = address(merkleTreeHook).addressToBytes32();
+
+        fixturePrefix(checkpointIndex, merkleTreeAddress, messageId, proof);
+
         return
             abi.encodePacked(
-                address(merkleTreeHook).addressToBytes32(),
+                merkleTreeAddress,
                 checkpointIndex,
-                message.id(),
-                merkleTreeHook.proof(),
+                messageId,
+                proof,
                 checkpointIndex
             );
     }
@@ -164,15 +260,24 @@ contract MessageIdMultisigIsmTest is AbstractMultisigIsmTest {
         mailbox.setRequiredHook(address(noopHook));
     }
 
+    function fixturePrefix(
+        bytes32 root,
+        uint32 index,
+        bytes32 merkleTreeAddress
+    ) internal {
+        vm.serializeBytes32(prefixKey, "root", root);
+        vm.serializeUint(prefixKey, "signedIndex", uint256(index));
+        vm.serializeBytes32(prefixKey, "merkleTree", merkleTreeAddress);
+    }
+
     function metadataPrefix(
         bytes memory
-    ) internal view override returns (bytes memory) {
+    ) internal override returns (bytes memory metadata) {
         (bytes32 root, uint32 index) = merkleTreeHook.latestCheckpoint();
-        return
-            abi.encodePacked(
-                address(merkleTreeHook).addressToBytes32(),
-                root,
-                index
-            );
+        bytes32 merkleTreeAddress = address(merkleTreeHook).addressToBytes32();
+
+        fixturePrefix(root, index, merkleTreeAddress);
+
+        return abi.encodePacked(merkleTreeAddress, root, index);
     }
 }