Update gas oracles, deploy agents (#3755)

### Description

- Updates gas oracles to make messages to Scroll slightly cheaper
- After running into issues configuring non-EVM remotes in the IGP / gas
oracle tooling (see
https://discord.com/channels/935678348330434570/1238439687165579264),
added a temporary band aid of just try/catching. We'll want to fix this
at some point in the future
- Deploys agents with a new image
- Updates the agent JSON configs with some params we were running the
agents with in prod via env vars
- Adjusts kathy frequency on testnet to send every 1.5hrs instead of
7hrs
- Stops running eclipsetestnet agents - apparently there was a reset

### Drive-by changes

Some drive-by logging changes

### Related issues

n/a

### Backward compatibility

<!--
Are these changes backward compatible? Are there any infrastructure
implications, e.g. changes that would prohibit deploying older commits
using this infra tooling?

Yes/No
-->

### Testing

<!--
What kind of testing have these changes undergone?

None/Manual/Unit Tests
-->

rust/agents/relayer/src/msg/metadata/aggregation.rs

@@ -108,7 +108,7 @@ impl AggregationIsmMetadataBuilder {
 
         let metas_and_gas_count = metas_and_gas.len();
         if metas_and_gas_count < threshold {
-            info!(?err_isms, %metas_and_gas_count, %threshold, message_id=message.id().to_string(), "Could not fetch all metadata, ISM metadata count did not reach aggregation threshold");
+            info!(?err_isms, %metas_and_gas_count, %threshold, message_id=?message.id(), "Could not fetch all metadata, ISM metadata count did not reach aggregation threshold");
             return None;
         }
         Some(Self::n_cheapest_metas(metas_and_gas, threshold))

rust/agents/relayer/src/msg/pending_message.rs

@@ -136,7 +136,7 @@ impl PendingOperation for PendingMessage {
         self.app_context.clone()
     }
 
-    #[instrument(skip(self), ret, fields(id=%self.id()), level = "debug")]
+    #[instrument(skip(self), ret, fields(id=?self.id()), level = "debug")]
     async fn prepare(&mut self) -> PendingOperationResult {
         make_op_try!(|| self.on_reprepare());
 
@@ -232,11 +232,10 @@ impl PendingOperation for PendingMessage {
         // Go ahead and attempt processing of message to destination chain.
         debug!(
             ?gas_limit,
+            ?tx_cost_estimate,
             "Gas payment requirement met, ready to process message"
         );
 
-        let gas_limit = tx_cost_estimate.gas_limit;
-
         if let Some(max_limit) = self.ctx.transaction_gas_limit {
             if gas_limit > max_limit {
                 info!("Message delivery estimated gas exceeds max gas limit");

rust/config/mainnet_config.json

@@ -614,7 +614,7 @@
         }
       ],
       "index": {
-        "chunk": 50,
+        "chunk": 25,
         "from": 58419500
       },
       "interchainGasPaymaster": "0x27ae52298e5b53b34b7ae0ca63e05845c31e1f59",
@@ -1037,6 +1037,7 @@
       "fallbackRoutingHook": "0xDa7cECb05C4aeB02c1aFDE277d4306a2da7Bd762",
       "gasCurrencyCoinGeckoId": "ethereum",
       "index": {
+        "chunk": 999,
         "from": 271840
       },
       "interchainAccountIsm": "0xb89c6ED617f5F46175E41551350725A09110bbCE",
@@ -1091,7 +1092,7 @@
       "domainRoutingIsmFactory": "0x1052eF3419f26Bec74Ed7CEf4a4FA6812Bc09908",
       "gasCurrencyCoinGeckoId": "tomochain",
       "index": {
-        "chunk": 1000,
+        "chunk": 999,
         "from": 73573878
       },
       "interchainAccountIsm": "0xD1E267d2d7876e97E217BfE61c34AB50FEF52807",

rust/config/testnet_config.json

@@ -79,6 +79,7 @@
       "domainId": 97,
       "fallbackRoutingHook": "0x2670ED2EC08cAd135307556685a96bD4c16b007b",
       "index": {
+        "chunk": 1000,
         "from": 34323977
       },
       "interchainAccountIsm": "0xa9D8Ec959F34272B1a56D09AF00eeee58970d3AE",

rust/hyperlane-core/src/types/primitive_types.rs

@@ -5,7 +5,7 @@
 
 use std::{ops::Mul, str::FromStr};
 
-use bigdecimal::BigDecimal;
+use bigdecimal::{BigDecimal, RoundingMode};
 use borsh::{BorshDeserialize, BorshSerialize};
 use fixed_hash::impl_fixed_hash_conversions;
 use num_traits::Zero;
@@ -354,7 +354,7 @@ impl FixedPointNumber {
 
     /// Round up to the nearest integer
     pub fn ceil_to_integer(&self) -> Self {
-        Self(self.0.with_scale(0))
+        Self(self.0.with_scale_round(0, RoundingMode::Ceiling))
     }
 
     /// Ceil
@@ -428,3 +428,28 @@ impl FromStr for FixedPointNumber {
         Ok(Self(BigDecimal::from_str(s)?))
     }
 }
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn test_fixed_point_number_ceil_to_integer() {
+        use super::FixedPointNumber;
+        use std::str::FromStr;
+
+        // Ceil a non-integer value
+        assert_eq!(
+            FixedPointNumber::from_str("1234.005")
+                .unwrap()
+                .ceil_to_integer(),
+            FixedPointNumber::from_str("1235").unwrap(),
+        );
+
+        // Don't change an integer value
+        assert_eq!(
+            FixedPointNumber::from_str("1234")
+                .unwrap()
+                .ceil_to_integer(),
+            FixedPointNumber::from_str("1234").unwrap(),
+        );
+    }
+}

typescript/infra/config/environments/mainnet3/agent.ts

@@ -22,6 +22,7 @@ import { supportedChainNames } from './supportedChainNames.js';
 import { validatorChainConfig } from './validators.js';
 import ancient8EthereumUsdcAddresses from './warp/ancient8-USDC-addresses.json';
 import arbitrumTIAAddresses from './warp/arbitrum-TIA-addresses.json';
+import arbitrumNeutronEclipAddresses from './warp/arbitrum-neutron-eclip-addresses.json';
 import inevmEthereumUsdcAddresses from './warp/inevm-USDC-addresses.json';
 import inevmEthereumUsdtAddresses from './warp/inevm-USDT-addresses.json';
 import injectiveInevmInjAddresses from './warp/injective-inevm-addresses.json';
@@ -202,15 +203,15 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '3012392-20240507-130024',
+      tag: 'c9c5d37-20240510-014327',
     },
     gasPaymentEnforcement: gasPaymentEnforcement,
     metricAppContexts,
   },
   validators: {
     docker: {
       repo,
-      tag: 'a2d6af6-20240422-164135',
+      tag: 'c9c5d37-20240510-014327',
     },
     rpcConsensusType: RpcConsensusType.Quorum,
     chains: validatorChainConfig(Contexts.Hyperlane),
@@ -219,7 +220,7 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: 'a2d6af6-20240422-164135',
+      tag: 'c9c5d37-20240510-014327',
     },
   },
 };
@@ -233,7 +234,7 @@ const releaseCandidate: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '3012392-20240507-130024',
+      tag: 'c9c5d37-20240510-014327',
     },
     // We're temporarily (ab)using the RC relayer as a way to increase
     // message throughput.
@@ -244,7 +245,7 @@ const releaseCandidate: RootAgentConfig = {
   validators: {
     docker: {
       repo,
-      tag: 'a2d6af6-20240422-164135',
+      tag: 'c9c5d37-20240510-014327',
     },
     rpcConsensusType: RpcConsensusType.Quorum,
     chains: validatorChainConfig(Contexts.ReleaseCandidate),
@@ -264,14 +265,15 @@ const neutron: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: 'a2d6af6-20240422-164135',
+      tag: 'c9c5d37-20240510-014327',
     },
     gasPaymentEnforcement: [
       {
         type: GasPaymentEnforcementPolicyType.None,
         matchingList: [
           ...routerMatchingList(mantaTIAAddresses),
           ...routerMatchingList(arbitrumTIAAddresses),
+          ...routerMatchingList(arbitrumNeutronEclipAddresses),
         ],
       },
       ...gasPaymentEnforcement,
@@ -285,6 +287,10 @@ const neutron: RootAgentConfig = {
         name: 'arbitrum_tia',
         matchingList: routerMatchingList(arbitrumTIAAddresses),
       },
+      {
+        name: 'arbitrum_neutron_eclip',
+        matchingList: routerMatchingList(arbitrumNeutronEclipAddresses),
+      },
     ],
   },
 };

typescript/infra/config/environments/mainnet3/helloworld.ts

@@ -6,7 +6,7 @@ import {
 } from '../../../src/config/helloworld/types.js';
 import { Contexts } from '../../contexts.js';
 
-import { environment } from './chains.js';
+import { environment, ethereumChainNames } from './chains.js';
 import hyperlaneAddresses from './helloworld/hyperlane/addresses.json';
 import rcAddresses from './helloworld/rc/addresses.json';
 
@@ -22,7 +22,7 @@ export const hyperlane: HelloWorldConfig = {
     namespace: environment,
     runConfig: {
       mode: HelloWorldKathyRunMode.Service,
-      fullCycleTime: 1000 * 60 * 60 * 24 * 5, // every 5 days, 13 * 12 messages = 156 messages is little less than once an hour
+      fullCycleTime: 1000 * 60 * 60 * 24 * 7, // every 7 days, 15 * 14 messages = 210 messages is every ~45 mins
     },
     messageSendTimeout: 1000 * 60 * 8, // 8 min
     messageReceiptTimeout: 1000 * 60 * 20, // 20 min

typescript/infra/config/environments/mainnet3/igp.ts

@@ -50,7 +50,7 @@ function getTokenExchangeRate(local: ChainName, remote: ChainName): BigNumber {
 
 const storageGasOracleConfig: AllStorageGasOracleConfigs =
   getAllStorageGasOracleConfigs(
-    ethereumChainNames,
+    supportedChainNames,
     gasPrices,
     getTokenExchangeRate,
     (local) => parseFloat(tokenPrices[local]),

typescript/infra/config/environments/mainnet3/warp/arbitrum-neutron-eclip-addresses.json

@@ -0,0 +1,8 @@
+{
+  "neutron": {
+    "router": "neutron1dvzvf870mx9uf65uqhx40yzx9gu4xlqqq2pnx362a0ndmustww3smumrf5"
+  },
+  "arbitrum": {
+    "router": "0x93ca0d85837FF83158Cd14D65B169CdB223b1921"
+  }
+}

typescript/infra/config/environments/testnet4/agent.ts

@@ -34,7 +34,7 @@ export const hyperlaneContextAgentChainConfig: AgentChainConfig = {
   [Role.Validator]: {
     alfajores: true,
     bsctestnet: true,
-    eclipsetestnet: true,
+    eclipsetestnet: false,
     fuji: true,
     plumetestnet: true,
     scrollsepolia: true,
@@ -44,7 +44,7 @@ export const hyperlaneContextAgentChainConfig: AgentChainConfig = {
   [Role.Relayer]: {
     alfajores: true,
     bsctestnet: true,
-    eclipsetestnet: true,
+    eclipsetestnet: false,
     fuji: true,
     plumetestnet: true,
     scrollsepolia: true,
@@ -95,7 +95,7 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '17ac515-20240402-171933',
+      tag: 'c9c5d37-20240510-014327',
     },
     blacklist: [
       ...releaseCandidateHelloworldMatchingList,
@@ -124,15 +124,15 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '17ac515-20240402-171933',
+      tag: 'c9c5d37-20240510-014327',
     },
     chains: validatorChainConfig(Contexts.Hyperlane),
   },
   scraper: {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '17ac515-20240402-171933',
+      tag: 'c9c5d37-20240510-014327',
     },
   },
 };
@@ -146,7 +146,7 @@ const releaseCandidate: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '17ac515-20240402-171933',
+      tag: 'c9c5d37-20240510-014327',
     },
     whitelist: [...releaseCandidateHelloworldMatchingList],
     gasPaymentEnforcement,
@@ -156,7 +156,7 @@ const releaseCandidate: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '17ac515-20240402-171933',
+      tag: 'c9c5d37-20240510-014327',
     },
     chains: validatorChainConfig(Contexts.ReleaseCandidate),
   },

typescript/infra/config/environments/testnet4/aw-validators/hyperlane.json

@@ -13,9 +13,6 @@
       "0x1f030345963c54ff8229720dd3a711c15c554aeb"
     ]
   },
-  "eclipsetestnet": {
-    "validators": ["0xf344f34abca9a444545b5295066348a0ae22dda3"]
-  },
   "fuji": {
     "validators": [
       "0xd8154f73d04cc7f7f0c332793692e6e6f6b2402e",

typescript/infra/config/environments/testnet4/helloworld.ts

@@ -6,7 +6,7 @@ import {
 } from '../../../src/config/helloworld/types.js';
 import { Contexts } from '../../contexts.js';
 
-import { environment } from './chains.js';
+import { environment, ethereumChainNames } from './chains.js';
 import hyperlaneAddresses from './helloworld/hyperlane/addresses.json';
 import rcAddresses from './helloworld/rc/addresses.json';
 
@@ -22,7 +22,7 @@ export const hyperlaneHelloworld: HelloWorldConfig = {
     namespace: environment,
     runConfig: {
       mode: HelloWorldKathyRunMode.Service,
-      fullCycleTime: 1000 * 60 * 60 * 24 * 6, // every 6 days. At 12 chains it 12 * 11 messages = 132 messages its a bit less than once an hour
+      fullCycleTime: 1000 * 60 * 60 * 24 * 2, // 2 days, 6 * 5 = 30 permutations, so ~1.5 hours per permutation
     },
     messageSendTimeout: 1000 * 60 * 10, // 10 min
     messageReceiptTimeout: 1000 * 60 * 20, // 20 min

typescript/infra/src/config/gas-oracle.ts

@@ -146,7 +146,7 @@ function getMinUsdCost(local: ChainName, remote: ChainName): number {
     optimism: 0.5,
     polygonzkevm: 0.5,
     // Scroll is more expensive than the rest due to higher L1 fees
-    scroll: 2,
+    scroll: 1.5,
     // Nexus adjustment
     neutron: 0.5,
   };

typescript/infra/src/govern/HyperlaneAppGovernor.ts

@@ -89,7 +89,7 @@ export abstract class HyperlaneAppGovernor<
     ): Promise<boolean> => {
       if (calls.length > 0) {
         console.log(
-          `> ${calls.length} calls will be submitted via ${submissionType}`,
+          `> ${calls.length} calls will be submitted via ${SubmissionType[submissionType]}`,
         );
         calls.map((c) =>
           console.log(`> > ${c.description} (to: ${c.to} data: ${c.data})`),

typescript/sdk/src/gas/HyperlaneIgpChecker.ts

@@ -88,8 +88,14 @@ export class HyperlaneIgpChecker extends HyperlaneAppChecker<
         );
         expectedOverhead = 0;
       }
-
-      const remoteId = this.multiProvider.getDomainId(remote);
+      // TODO: add back support for non-EVM remotes.
+      const remoteId = this.multiProvider.tryGetDomainId(remote);
+      if (remoteId === null) {
+        this.app.logger.warn(
+          `Skipping checking IGP ${local} -> ${remote}. Expected if the remote is a non-EVM chain.`,
+        );
+        continue;
+      }
       const existingOverhead = await defaultIsmIgp.destinationGasLimit(
         remoteId,
         0,
@@ -128,7 +134,14 @@ export class HyperlaneIgpChecker extends HyperlaneAppChecker<
       Object.keys(this.configMap[local].oracleConfig ?? {}),
     );
     for (const remote of remotes) {
-      const remoteId = this.multiProvider.getDomainId(remote);
+      // TODO: add back support for non-EVM remotes.
+      const remoteId = this.multiProvider.tryGetDomainId(remote);
+      if (remoteId === null) {
+        this.app.logger.warn(
+          `Skipping checking IGP ${local} -> ${remote}. Expected if the remote is a non-EVM chain.`,
+        );
+        continue;
+      }
       const destinationGasConfigs = await igp.destinationGasConfigs(remoteId);
       const actualGasOracle = destinationGasConfigs.gasOracle;
       const expectedGasOracle = coreContracts.storageGasOracle.address;