Problem: testground test case not fast enough

Solution:
- patch cronosd temporarily to disable nonce checking
- optimise the load generator

go.mod

@@ -276,7 +276,7 @@ replace (
 	github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.4.2
 	github.com/ethereum/go-ethereum => github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7
 	// block-stm branch
-	github.com/evmos/ethermint => github.com/crypto-org-chain/ethermint v0.6.1-0.20240626122608-773438f47b95
+	github.com/evmos/ethermint => github.com/yihuang/ethermint v0.6.1-0.20240627093737-b98aaf7a1d5a
 	// Fix upstream GHSA-h395-qcrw-5vmq and GHSA-3vp4-m3rf-835h vulnerabilities.
 	// TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409
 	github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.9.0

go.sum

@@ -430,8 +430,6 @@ github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240626040048-36295f051595
 github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240626040048-36295f051595/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg=
 github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240626040048-36295f051595 h1:MPKv1EzM16dx+HzkJowgb9PrlbatRlgFYqk1IucsL2s=
 github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240626040048-36295f051595/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY=
-github.com/crypto-org-chain/ethermint v0.6.1-0.20240626122608-773438f47b95 h1:3GAsuRgeQfumWPu7fxW9EbD2/vUVJomzYpE+jItOF24=
-github.com/crypto-org-chain/ethermint v0.6.1-0.20240626122608-773438f47b95/go.mod h1:z5zfkmH6XonIpRzMzFoZU+QSBkXSe5GdRvRWiK9fiA8=
 github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde h1:sQIHTJfVt5VTrF7po9eZiFkZiPjlHbFvnXtGCOoBjNM=
 github.com/crypto-org-chain/go-block-stm v0.0.0-20240408011717-9f11af197bde/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7 h1:V43F3JFcqG4MUThf9W/DytnPblpR6CcaLBw2Wx6zTgE=
@@ -1160,6 +1158,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/yihuang/ethermint v0.6.1-0.20240627093737-b98aaf7a1d5a h1:YLz7XqovXjSNsRf/veTPDmh0MSEvM9VrfZt2B3V4QCc=
+github.com/yihuang/ethermint v0.6.1-0.20240627093737-b98aaf7a1d5a/go.mod h1:z5zfkmH6XonIpRzMzFoZU+QSBkXSe5GdRvRWiK9fiA8=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

gomod2nix.toml

@@ -262,9 +262,9 @@ schema = 3
     hash = "sha256-lE4G5FaRb3MVi9FFVn+WlwsSTOB4SbjmVboKyQ5yB0A="
     replaced = "github.com/crypto-org-chain/go-ethereum"
   [mod."github.com/evmos/ethermint"]
-    version = "v0.6.1-0.20240626122608-773438f47b95"
-    hash = "sha256-L/hj92fMu/+5XBb1pcA6mk6oNyN7HidC9lT+nPQcPu4="
-    replaced = "github.com/crypto-org-chain/ethermint"
+    version = "v0.6.1-0.20240627093737-b98aaf7a1d5a"
+    hash = "sha256-UkK9envr8PITikH5zC7Hr875snEbjR4VyQq/BQX3l5g="
+    replaced = "github.com/yihuang/ethermint"
   [mod."github.com/fatih/color"]
     version = "v1.16.0"
     hash = "sha256-Aq/SM28aPJVzvapllQ64R/DM4aZ5CHPewcm/AUJPyJQ="

nix/testground-cronosd.patch

@@ -0,0 +1,12 @@
+diff --git a/app/app.go b/app/app.go
+index 678ccc0a..ba64a7fe 100644
+--- a/app/app.go
++++ b/app/app.go
+@@ -1060,6 +1060,7 @@ func (app *App) setAnteHandler(txConfig client.TxConfig, maxGasWanted uint64, bl
+ 		},
+ 		ExtraDecorators:   []sdk.AnteDecorator{blockAddressDecorator},
+ 		PendingTxListener: app.onPendingTx,
++		UnsafeUnorderedTx: true,
+ 	}
+
+ 	anteHandler, err := evmante.NewAnteHandler(options)

nix/testground-image.nix

@@ -1,10 +1,17 @@
 { dockerTools, cronos-matrix, testground-testcase }:
+let
+  patched-cronosd = cronos-matrix.cronosd.overrideAttrs (oldAttrs: {
+    patches = oldAttrs.patches ++ [
+      ./testground-cronosd.patch
+    ];
+  });
+in
 dockerTools.buildLayeredImage {
   name = "cronos-testground";
   created = "now";
   contents = [
     testground-testcase
-    cronos-matrix.cronosd
+    patched-cronosd
   ];
   config = {
     Expose = [ 9090 26657 26656 1317 26658 26660 26659 30000 ];

testground/benchmark/benchmark/context.py

@@ -4,7 +4,7 @@
 from .params import RunParams, run_params
 from .sync import SyncService
 
-LEADER_GLOBAL_SEQUENCE = 1
+LEADER_SEQUENCE = 1
 
 
 class Context:
@@ -103,12 +103,24 @@ def record_stage_end(self, name: str):
 
     @property
     def is_leader(self) -> bool:
-        return self.global_seq == LEADER_GLOBAL_SEQUENCE
+        return self.global_seq == LEADER_SEQUENCE
+
+    @property
+    def is_fullnode_leader(self) -> bool:
+        return not self.is_validator and self.group_seq == LEADER_SEQUENCE
+
+    @property
+    def is_validator_leader(self) -> bool:
+        return self.is_validator and self.group_seq == LEADER_SEQUENCE
 
     @property
     def is_validator(self) -> bool:
         return self.params.is_validator
 
+    @property
+    def is_fullnode(self) -> bool:
+        return not self.params.is_validator
+
     def __enter__(self):
         return self
 

testground/benchmark/benchmark/main.py

@@ -1,11 +1,15 @@
 import os
 import subprocess
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+import web3
 
 from .cli import ChainCommand
 from .context import Context
 from .peer import bootstrap
-from .sendtx import sendtx
-from .utils import wait_for_block, wait_for_port
+from .sendtx import fund_test_accounts, sendtx
+from .utils import export_eth_account, wait_for_block, wait_for_port
 
 CRONOSD_PATH = "/bin/cronosd"
 
@@ -20,20 +24,47 @@ def entrypoint(ctx: Context):
     cli = ChainCommand(CRONOSD_PATH)
 
     # build the genesis file collectively, and setup the network topology
-    peer = bootstrap(ctx, cli)
+    bootstrap(ctx, cli)
 
     # start the node
-    kwargs = {"stdout": subprocess.DEVNULL}
-    if ctx.is_leader:
-        del kwargs["stdout"]
-    proc = subprocess.Popen([CRONOSD_PATH, "start"], **kwargs)
+    logfile = Path(ctx.params.test_outputs_path) / "node.log"
+    proc = subprocess.Popen(
+        [CRONOSD_PATH, "start"],
+        stdout=open(logfile, "ab", buffering=0),
+    )
 
     wait_for_port(26657)
     wait_for_port(8545)
     wait_for_block(cli, 1)
 
+    test_finish_entry = f"finish-test-{ctx.params.test_group_id}"
     if not ctx.is_validator:
-        sendtx(cli, peer)
+        w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545"))
+        assert w3.eth.chain_id == 777
+        genesis_account = export_eth_account(cli, "account")
+        accounts = fund_test_accounts(w3, genesis_account, ctx.params.num_accounts)
+        with ThreadPoolExecutor(max_workers=ctx.params.num_accounts) as executor:
+            futs = (
+                executor.submit(sendtx, w3, acct, ctx.params.num_txs)
+                for acct in accounts
+            )
+            for fut in as_completed(futs):
+                try:
+                    fut.result()
+                except Exception as e:
+                    print("test task failed", e)
+
+        print("finish test", ctx.group_seq)
+        ctx.sync.signal_and_wait(
+            test_finish_entry, ctx.params.test_group_instance_count
+        )
+
+    if ctx.is_fullnode_leader:
+        # collect output
+        w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545"))
+        for i in range(w3.eth.block_number):
+            blk = w3.eth.get_block(i)
+            print(i, len(blk.transactions), blk.timestamp)
 
     # halt after all tasks are done
     ctx.sync.signal_and_wait("halt", ctx.params.test_instance_count)
@@ -43,6 +74,7 @@ def entrypoint(ctx: Context):
         proc.wait(5)
     except subprocess.TimeoutExpired:
         pass
+
     ctx.record_success()
 
 

testground/benchmark/benchmark/params.py

@@ -107,8 +107,12 @@ def chain_id(self) -> str:
         return self.test_instance_params["chain_id"]
 
     @property
-    def halt_height(self) -> int:
-        return int(self.test_instance_params["halt_height"])
+    def num_accounts(self) -> int:
+        return int(self.test_instance_params["num_accounts"])
+
+    @property
+    def num_txs(self) -> int:
+        return int(self.test_instance_params["num_txs"])
 
 
 def run_params(env=None) -> RunParams:

testground/benchmark/benchmark/peer.py

@@ -30,42 +30,32 @@ def bootstrap(ctx: Context, cli) -> PeerPacket:
     )
     account_addr = cli("keys", "show", "account", "--address", keyring_backend="test")
     accounts = [
-        GenesisAccount(
-            address=validator_addr,
-            balance=VAL_INITIAL_AMOUNT,
-        ),
-        GenesisAccount(
-            address=account_addr,
-            balance=ACC_INITIAL_AMOUNT,
-        ),
+        GenesisAccount(address=validator_addr, balance=VAL_INITIAL_AMOUNT),
+        GenesisAccount(address=account_addr, balance=ACC_INITIAL_AMOUNT),
     ]
 
     node_id = cli("comet", "show-node-id")
     peer_id = f"{node_id}@{ip}:26656"
-    peer = PeerPacket(
+    current = PeerPacket(
         ip=str(ip),
         node_id=node_id,
         peer_id=peer_id,
         accounts=accounts,
     )
 
     if ctx.is_validator:
-        peer.gentx = gentx(cli, ctx.params.chain_id)
+        current.gentx = gentx(cli, ctx.params.chain_id)
 
     data = ctx.sync.publish_subscribe_simple(
-        "peers", peer.dict(), ctx.params.test_instance_count
+        "peers", current.dict(), ctx.params.test_instance_count
     )
     peers: List[PeerPacket] = [PeerPacket.model_validate(item) for item in data]
 
     config_path = Path.home() / ".cronos" / "config"
-    if ctx.is_leader:
+    if ctx.is_fullnode_leader:
         # prepare genesis file and publish
         for peer in peers:
             for account in peer.accounts:
-                if ctx.is_validator and account.address == validator_addr:
-                    # if leader is also validator, it's validator account is already
-                    # added in gentx
-                    continue
                 cli("genesis", "add-genesis-account", account.address, account.balance)
         collect_gen_tx(cli, peers)
         cli("genesis", "validate")
@@ -74,6 +64,7 @@ def bootstrap(ctx: Context, cli) -> PeerPacket:
             {
                 "consensus.params.block.max_gas": "81500000",
                 "app_state.evm.params.evm_denom": "basecro",
+                "app_state.feemarket.params.no_base_fee": True,
             },
         )
         ctx.sync.publish("genesis", genesis)
@@ -83,22 +74,26 @@ def bootstrap(ctx: Context, cli) -> PeerPacket:
         genesis_file.write_text(json.dumps(genesis))
         cli("genesis", "validate")
 
-    # update persistent_peers in config.toml
-    patch_toml(
-        config_path / "config.toml",
-        {
-            "p2p.persistent_peers": connect_all(peer, peers),
-            "mempool.recheck": "false",
-        },
-    )
-    patch_toml(
-        config_path / "app.toml",
-        {
-            "minimum-gas-prices": "0basecro",
-        },
-    )
+    # update persistent_peers and other configs in config.toml
+    config_patch = {
+        "p2p.persistent_peers": connect_all(current, peers),
+        "mempool.recheck": "false",
+        "mempool.size": 50000,
+        "consensus.timeout_commit": "2s",
+    }
+    if ctx.is_validator:
+        config_patch["tx_index.indexer"] = "null"
+
+    app_patch = {
+        "minimum-gas-prices": "0basecro",
+        "index-events": ["ethereum_tx.ethereumTxHash"],
+        "memiavl.enable": True,
+    }
+
+    patch_toml(config_path / "config.toml", config_patch)
+    patch_toml(config_path / "app.toml", app_patch)
 
-    return peer
+    return current
 
 
 def gentx(cli, chain_id):

testground/benchmark/benchmark/sendtx.py

@@ -1,38 +1,56 @@
+import time
+
 import web3
 from eth_account import Account
 
-from .types import PeerPacket
 from .utils import send_transaction
 
-TX_AMOUNT = 1000
+TEST_AMOUNT = 1000000000000000000
+GAS_PRICE = 1000000000
+
+
+def fund_test_accounts(w3, from_account, num_accounts) -> [Account]:
+    accounts = []
+    for i in range(num_accounts):
+        acct = Account.create()
+        tx = {
+            "to": acct.address,
+            "value": TEST_AMOUNT,
+            "gas": 21000,
+            "gasPrice": GAS_PRICE,
+        }
+        receipt = send_transaction(w3, tx, from_account, wait=True)
+        assert receipt.status == 1
+        print("fund test account", acct.address, "balance", TEST_AMOUNT)
+        accounts.append(acct)
+    return accounts
 
 
-def sendtx(cli, peer: PeerPacket):
-    w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545"))
-    assert w3.eth.chain_id == 777
-    acct = export_eth_account(cli, "account")
+def sendtx(w3: web3.Web3, acct: Account, tx_amount: int):
     print("test address", acct.address, "balance", w3.eth.get_balance(acct.address))
 
-    nonce = w3.eth.get_transaction_count(acct.address)
-    for i in range(TX_AMOUNT):
+    initial_nonce = w3.eth.get_transaction_count(acct.address)
+    nonce = initial_nonce
+    while nonce < initial_nonce + tx_amount:
         tx = {
             "to": "0x0000000000000000000000000000000000000000",
             "value": 1,
             "nonce": nonce,
             "gas": 21000,
+            "gasPrice": GAS_PRICE,
         }
         try:
             send_transaction(w3, tx, acct, wait=False)
         except ValueError as e:
-            if "invalid nonce" in str(e):
-                # reset nonce and continue
-                nonce = w3.eth.get_transaction_count(acct.address)
+            msg = str(e)
+            if "invalid nonce" in msg:
+                print("invalid nonce and retry", nonce)
+                time.sleep(1)
                 continue
-            raise
-        nonce += 1
+            if "tx already in mempool" not in msg:
+                raise
 
+        nonce += 1
 
-def export_eth_account(cli, name: str) -> Account:
-    return Account.from_key(
-        cli("keys", "unsafe-export-eth-key", name, keyring_backend="test")
-    )
+        if nonce % 100 == 0:
+            print(f"{acct.address} sent {nonce} transactions")

testground/benchmark/benchmark/sync.py

@@ -70,6 +70,9 @@ def _request(self, name, payload):
         )
 
         rsp = cond.wait()
+        if rsp is None:
+            # connection closed
+            return
         del self._handlers[id]
         assert not rsp.get("error"), rsp
         return rsp