From 040bdbfd32e9842ce25f9cfa9b480029945da885 Mon Sep 17 00:00:00 2001 From: Mattie Conover Date: Mon, 2 Oct 2023 10:05:20 -0700 Subject: [PATCH] Streamline Rust Agent Config Shapes (#2659) ### Description Continues the updates to the rust config shapes by updating deployment and runtime expectations. This PR also attempts to rely on the new single source of truth created by the schema in the SDK as much as reasonably possible which helped delete more code but also give some guarantees of consistency. THIS IS A BREAKING CHANGE! It changes the config shapes the agents want and we should not merge this until we are ready. Fixes #2215 --------- Co-authored-by: Guillaume Bouvignies Co-authored-by: Yorke Rhodes Co-authored-by: Guillaume Bouvignies --- .gitignore | 1 + rust/Cargo.lock | 1 + rust/agents/relayer/Cargo.toml | 1 + .../agents/relayer/src/msg/gas_payment/mod.rs | 21 +- .../relayer/src/settings/matching_list.rs | 83 +++- rust/agents/relayer/src/settings/mod.rs | 395 ++-------------- rust/agents/scraper/src/settings.rs | 71 +-- rust/agents/validator/src/settings.rs | 127 +---- rust/chains/hyperlane-ethereum/src/config.rs | 95 ---- .../hyperlane-fuel/src/trait_builder.rs | 29 +- .../hyperlane-sealevel/src/trait_builder.rs | 32 +- rust/config/test_sealevel_config.json | 46 +- rust/helm/agent-common/templates/_helpers.tpl | 20 +- rust/helm/hyperlane-agent/README.md | 44 -- .../hyperlane-agent/templates/configmap.yaml | 10 +- .../templates/external-secret.yaml | 12 +- .../templates/relayer-external-secret.yaml | 6 +- .../templates/relayer-statefulset.yaml | 9 +- .../templates/scraper-external-secret.yaml | 2 +- .../templates/scraper-statefulset.yaml | 9 +- .../templates/validator-configmap.yaml | 6 +- .../templates/validator-external-secret.yaml | 14 +- rust/helm/hyperlane-agent/values.yaml | 37 +- .../src/settings/deprecated_parser.rs | 435 ------------------ .../src/settings/loader/arguments.rs | 15 +- .../src/settings/loader/case_adapter.rs | 66 +++ .../settings/loader/deprecated_arguments.rs | 343 -------------- .../src/settings/loader/environment.rs | 38 +- .../hyperlane-base/src/settings/loader/mod.rs | 132 +++--- rust/hyperlane-base/src/settings/mod.rs | 30 +- .../src/settings/parser/json_value_parser.rs | 37 +- .../hyperlane-base/src/settings/parser/mod.rs | 146 +++--- rust/hyperlane-base/tests/chain_config.rs | 6 +- rust/hyperlane-core/src/config/config_path.rs | 4 +- .../testwarproute/program-ids.json | 8 +- rust/utils/run-locally/src/main.rs | 61 +-- rust/utils/run-locally/src/program.rs | 41 +- .../config/environments/mainnet2/agent.ts | 16 +- .../config/environments/mainnet2/funding.ts | 4 +- .../environments/mainnet2/helloworld.ts | 6 +- .../config/environments/mainnet2/index.ts | 4 +- .../environments/mainnet2/liquidityLayer.ts | 4 +- .../infra/config/environments/test/agent.ts | 10 +- .../config/environments/testnet3/agent.ts | 16 +- .../config/environments/testnet3/funding.ts | 4 +- .../environments/testnet3/helloworld.ts | 6 +- .../config/environments/testnet3/index.ts | 4 +- .../environments/testnet3/middleware.ts | 4 +- .../templates/external-secret.yaml | 4 +- .../templates/env-var-external-secret.yaml | 4 +- .../templates/env-var-external-secret.yaml | 4 +- typescript/infra/scripts/agents/utils.ts | 8 + .../funding/fund-keys-from-deployer.ts | 8 +- typescript/infra/scripts/helloworld/kathy.ts | 10 +- typescript/infra/scripts/helloworld/utils.ts | 6 +- typescript/infra/scripts/utils.ts | 5 +- typescript/infra/src/agents/aws/key.ts | 6 +- typescript/infra/src/agents/index.ts | 27 +- typescript/infra/src/config/agent/agent.ts | 47 +- typescript/infra/src/config/agent/index.ts | 6 +- typescript/infra/src/config/agent/relayer.ts | 83 +--- typescript/infra/src/config/agent/scraper.ts | 12 +- .../infra/src/config/agent/validator.ts | 58 ++- typescript/infra/src/config/chain.ts | 12 +- typescript/infra/src/config/environment.ts | 4 +- typescript/infra/src/config/funding.ts | 4 +- typescript/infra/src/config/helloworld.ts | 4 +- typescript/infra/src/config/middleware.ts | 4 +- typescript/infra/src/deployment/deploy.ts | 4 +- typescript/infra/tsconfig.json | 2 +- typescript/sdk/src/index.ts | 20 +- .../sdk/src/metadata/agentConfig.test.ts | 28 +- typescript/sdk/src/metadata/agentConfig.ts | 248 +++------- .../sdk/src/metadata/chainMetadataTypes.ts | 12 +- typescript/sdk/src/metadata/matchingList.ts | 2 +- typescript/sdk/tsconfig.json | 2 +- typescript/token/tsconfig.json | 0 typescript/utils/tsconfig.json | 2 +- 78 files changed, 833 insertions(+), 2314 deletions(-) delete mode 100644 rust/helm/hyperlane-agent/README.md delete mode 100644 rust/hyperlane-base/src/settings/deprecated_parser.rs create mode 100644 rust/hyperlane-base/src/settings/loader/case_adapter.rs delete mode 100644 rust/hyperlane-base/src/settings/loader/deprecated_arguments.rs create mode 100644 typescript/token/tsconfig.json diff --git a/.gitignore b/.gitignore index 4989ff4f48..f9892417af 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ yarn-error.log **/*.ignore .vscode +tsconfig.editor.json diff --git a/rust/Cargo.lock b/rust/Cargo.lock index adef287e76..2ca08007e4 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -6065,6 +6065,7 @@ dependencies = [ "async-trait", "backoff", "config", + "convert_case 0.6.0", "derive-new", "derive_more", "enum_dispatch", diff --git a/rust/agents/relayer/Cargo.toml b/rust/agents/relayer/Cargo.toml index 1c69ce5988..406bf96925 100644 --- a/rust/agents/relayer/Cargo.toml +++ b/rust/agents/relayer/Cargo.toml @@ -13,6 +13,7 @@ version.workspace = true async-trait.workspace = true backoff.workspace = true config.workspace = true +convert_case.workspace = true derive-new.workspace = true derive_more.workspace = true enum_dispatch.workspace = true diff --git a/rust/agents/relayer/src/msg/gas_payment/mod.rs b/rust/agents/relayer/src/msg/gas_payment/mod.rs index fa4497c6f8..4ed30c9572 100644 --- a/rust/agents/relayer/src/msg/gas_payment/mod.rs +++ b/rust/agents/relayer/src/msg/gas_payment/mod.rs @@ -2,20 +2,20 @@ use std::fmt::Debug; use async_trait::async_trait; use eyre::Result; -use tracing::{debug, error, trace}; - use hyperlane_base::db::HyperlaneRocksDB; use hyperlane_core::{ GasPaymentKey, HyperlaneMessage, InterchainGasExpenditure, InterchainGasPayment, TxCostEstimate, TxOutcome, U256, }; - -use crate::msg::gas_payment::policies::GasPaymentPolicyOnChainFeeQuoting; -use crate::settings::{ - matching_list::MatchingList, GasPaymentEnforcementConf, GasPaymentEnforcementPolicy, -}; +use tracing::{debug, error, trace}; use self::policies::{GasPaymentPolicyMinimum, GasPaymentPolicyNone}; +use crate::{ + msg::gas_payment::policies::GasPaymentPolicyOnChainFeeQuoting, + settings::{ + matching_list::MatchingList, GasPaymentEnforcementConf, GasPaymentEnforcementPolicy, + }, +}; mod policies; @@ -148,12 +148,11 @@ mod test { H256, U256, }; + use super::GasPaymentEnforcer; use crate::settings::{ matching_list::MatchingList, GasPaymentEnforcementConf, GasPaymentEnforcementPolicy, }; - use super::GasPaymentEnforcer; - #[tokio::test] async fn test_empty_whitelist() { test_utils::run_test_db(|db| async move { @@ -195,7 +194,7 @@ mod test { test_utils::run_test_db(|db| async move { let hyperlane_db = HyperlaneRocksDB::new(&HyperlaneDomain::new_test_domain("test_no_match"), db); - let matching_list = serde_json::from_str(r#"[{"originDomain": 234}]"#).unwrap(); + let matching_list = serde_json::from_str(r#"[{"origindomain": 234}]"#).unwrap(); let enforcer = GasPaymentEnforcer::new( // Require a payment vec![GasPaymentEnforcementConf { @@ -339,7 +338,7 @@ mod test { let recipient_address = "0xbb000000000000000000000000000000000000bb"; let matching_list = serde_json::from_str( - &format!(r#"[{{"senderAddress": "{sender_address}", "recipientAddress": "{recipient_address}"}}]"#) + &format!(r#"[{{"senderaddress": "{sender_address}", "recipientaddress": "{recipient_address}"}}]"#) ).unwrap(); let enforcer = GasPaymentEnforcer::new( diff --git a/rust/agents/relayer/src/settings/matching_list.rs b/rust/agents/relayer/src/settings/matching_list.rs index d750c537c1..483b65ed9f 100644 --- a/rust/agents/relayer/src/settings/matching_list.rs +++ b/rust/agents/relayer/src/settings/matching_list.rs @@ -22,8 +22,7 @@ use serde::{ /// - wildcard "*" /// - single value in decimal or hex (must start with `0x`) format /// - list of values in decimal or hex format -#[derive(Debug, Deserialize, Default, Clone)] -#[serde(transparent)] +#[derive(Debug, Default, Clone)] pub struct MatchingList(Option>); #[derive(Debug, Clone, PartialEq)] @@ -63,6 +62,55 @@ impl Display for Filter { } } +struct MatchingListVisitor; +impl<'de> Visitor<'de> for MatchingListVisitor { + type Value = MatchingList; + + fn expecting(&self, fmt: &mut Formatter) -> fmt::Result { + write!(fmt, "an optional list of matching rules") + } + + fn visit_none(self) -> Result + where + E: Error, + { + Ok(MatchingList(None)) + } + + fn visit_some(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let list: Vec = deserializer.deserialize_seq(MatchingListArrayVisitor)?; + Ok(if list.is_empty() { + // this allows for empty matching lists to be treated as if no matching list was set + MatchingList(None) + } else { + MatchingList(Some(list)) + }) + } +} + +struct MatchingListArrayVisitor; +impl<'de> Visitor<'de> for MatchingListArrayVisitor { + type Value = Vec; + + fn expecting(&self, fmt: &mut Formatter) -> fmt::Result { + write!(fmt, "a list of matching rules") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut rules = seq.size_hint().map(Vec::with_capacity).unwrap_or_default(); + while let Some(rule) = seq.next_element::()? { + rules.push(rule); + } + Ok(rules) + } +} + struct FilterVisitor(PhantomData); impl<'de> Visitor<'de> for FilterVisitor { type Value = Filter; @@ -145,6 +193,15 @@ impl<'de> Visitor<'de> for FilterVisitor { } } +impl<'de> Deserialize<'de> for MatchingList { + fn deserialize(d: D) -> Result + where + D: Deserializer<'de>, + { + d.deserialize_option(MatchingListVisitor) + } +} + impl<'de> Deserialize<'de> for Filter { fn deserialize(d: D) -> Result where @@ -166,13 +223,13 @@ impl<'de> Deserialize<'de> for Filter { #[derive(Debug, Deserialize, Clone)] #[serde(tag = "type")] struct ListElement { - #[serde(default, rename = "originDomain")] + #[serde(default, rename = "origindomain")] origin_domain: Filter, - #[serde(default, rename = "senderAddress")] + #[serde(default, rename = "senderaddress")] sender_address: Filter, - #[serde(default, rename = "destinationDomain")] + #[serde(default, rename = "destinationdomain")] destination_domain: Filter, - #[serde(default, rename = "recipientAddress")] + #[serde(default, rename = "recipientaddress")] recipient_address: Filter, } @@ -266,7 +323,7 @@ mod test { #[test] fn basic_config() { - let list: MatchingList = serde_json::from_str(r#"[{"originDomain": "*", "senderAddress": "*", "destinationDomain": "*", "recipientAddress": "*"}, {}]"#).unwrap(); + let list: MatchingList = serde_json::from_str(r#"[{"origindomain": "*", "senderaddress": "*", "destinationdomain": "*", "recipientaddress": "*"}, {}]"#).unwrap(); assert!(list.0.is_some()); assert_eq!(list.0.as_ref().unwrap().len(), 2); let elem = &list.0.as_ref().unwrap()[0]; @@ -307,7 +364,7 @@ mod test { #[test] fn config_with_address() { - let list: MatchingList = serde_json::from_str(r#"[{"senderAddress": "0x9d4454B023096f34B160D6B654540c56A1F81688", "recipientAddress": "0x9d4454B023096f34B160D6B654540c56A1F81688"}]"#).unwrap(); + let list: MatchingList = serde_json::from_str(r#"[{"senderaddress": "0x9d4454B023096f34B160D6B654540c56A1F81688", "recipientaddress": "0x9d4454B023096f34B160D6B654540c56A1F81688"}]"#).unwrap(); assert!(list.0.is_some()); assert_eq!(list.0.as_ref().unwrap().len(), 1); let elem = &list.0.as_ref().unwrap()[0]; @@ -361,7 +418,7 @@ mod test { #[test] fn config_with_multiple_domains() { let whitelist: MatchingList = - serde_json::from_str(r#"[{"destinationDomain": ["13372", "13373"]}]"#).unwrap(); + serde_json::from_str(r#"[{"destinationdomain": ["13372", "13373"]}]"#).unwrap(); assert!(whitelist.0.is_some()); assert_eq!(whitelist.0.as_ref().unwrap().len(), 1); let elem = &whitelist.0.as_ref().unwrap()[0]; @@ -371,6 +428,12 @@ mod test { assert_eq!(elem.sender_address, Wildcard); } + #[test] + fn config_with_empty_list_is_none() { + let whitelist: MatchingList = serde_json::from_str(r#"[]"#).unwrap(); + assert!(whitelist.0.is_none()); + } + #[test] fn matches_empty_list() { let info = MatchInfo { @@ -388,7 +451,7 @@ mod test { #[test] fn supports_base58() { serde_json::from_str::( - r#"[{"originDomain":1399811151,"senderAddress":"DdTMkk9nuqH5LnD56HLkPiKMV3yB3BNEYSQfgmJHa5i7","destinationDomain":11155111,"recipientAddress":"0x6AD4DEBA8A147d000C09de6465267a9047d1c217"}]"#, + r#"[{"origindomain":1399811151,"senderaddress":"DdTMkk9nuqH5LnD56HLkPiKMV3yB3BNEYSQfgmJHa5i7","destinationdomain":11155111,"recipientaddress":"0x6AD4DEBA8A147d000C09de6465267a9047d1c217"}]"#, ).unwrap(); } } diff --git a/rust/agents/relayer/src/settings/mod.rs b/rust/agents/relayer/src/settings/mod.rs index 1d7cf2fd02..4071154752 100644 --- a/rust/agents/relayer/src/settings/mod.rs +++ b/rust/agents/relayer/src/settings/mod.rs @@ -6,13 +6,13 @@ use std::{collections::HashSet, path::PathBuf}; +use convert_case::Case; use derive_more::{AsMut, AsRef, Deref, DerefMut}; use eyre::{eyre, Context}; use hyperlane_base::{ impl_loadable_from_settings, settings::{ - deprecated_parser::DeprecatedRawSettings, - parser::{RawAgentConf, ValueParser}, + parser::{recase_json_value, RawAgentConf, ValueParser}, Settings, }, }; @@ -20,128 +20,11 @@ use hyperlane_core::{cfg_unwrap_all, config::*, HyperlaneDomain, U256}; use itertools::Itertools; use serde::Deserialize; use serde_json::Value; -use tracing::warn; use crate::settings::matching_list::MatchingList; pub mod matching_list; -/// Config for a GasPaymentEnforcementPolicy -#[derive(Debug, Clone, Default)] -pub enum GasPaymentEnforcementPolicy { - /// No requirement - all messages are processed regardless of gas payment - #[default] - None, - /// Messages that have paid a minimum amount will be processed - Minimum { payment: U256 }, - /// The required amount of gas on the foreign chain has been paid according - /// to on-chain fee quoting. - OnChainFeeQuoting { - gas_fraction_numerator: u64, - gas_fraction_denominator: u64, - }, -} - -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "camelCase")] -enum RawGasPaymentEnforcementPolicy { - None, - Minimum { - payment: Option, - }, - OnChainFeeQuoting { - /// Optional fraction of gas which must be paid before attempting to run - /// the transaction. Must be written as `"numerator / - /// denominator"` where both are integers. - #[serde(default = "default_gasfraction")] - gasfraction: String, - }, - #[serde(other)] - Unknown, -} - -impl FromRawConf for GasPaymentEnforcementPolicy { - fn from_config_filtered( - raw: RawGasPaymentEnforcementPolicy, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - use RawGasPaymentEnforcementPolicy::*; - match raw { - None => Ok(Self::None), - Minimum { payment } => Ok(Self::Minimum { - payment: payment - .ok_or_else(|| { - eyre!("Missing `payment` for Minimum gas payment enforcement policy") - }) - .into_config_result(|| cwp + "payment")? - .try_into() - .into_config_result(|| cwp + "payment")?, - }), - OnChainFeeQuoting { gasfraction } => { - let (numerator, denominator) = - gasfraction - .replace(' ', "") - .split_once('/') - .map(|(a, b)| (a.to_owned(), b.to_owned())) - .ok_or_else(|| eyre!("Invalid `gasfraction` for OnChainFeeQuoting gas payment enforcement policy; expected `numerator / denominator`")) - .into_config_result(|| cwp + "gasfraction")?; - - Ok(Self::OnChainFeeQuoting { - gas_fraction_numerator: numerator - .parse() - .into_config_result(|| cwp + "gasfraction")?, - gas_fraction_denominator: denominator - .parse() - .into_config_result(|| cwp + "gasfraction")?, - }) - } - Unknown => Err(eyre!("Unknown gas payment enforcement policy")) - .into_config_result(|| cwp.clone()), - } - } -} - -/// Config for gas payment enforcement -#[derive(Debug, Clone, Default)] -pub struct GasPaymentEnforcementConf { - /// The gas payment enforcement policy - pub policy: GasPaymentEnforcementPolicy, - /// An optional matching list, any message that matches will use this - /// policy. By default all messages will match. - pub matching_list: MatchingList, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawGasPaymentEnforcementConf { - #[serde(flatten)] - policy: Option, - #[serde(default)] - matching_list: Option, -} - -impl FromRawConf for GasPaymentEnforcementConf { - fn from_config_filtered( - raw: RawGasPaymentEnforcementConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - let policy = raw.policy - .ok_or_else(|| eyre!("Missing policy for gas payment enforcement config; required if a matching list is provided")) - .take_err(&mut err, || cwp.clone()).and_then(|r| { - r.parse_config(cwp).take_config_err(&mut err) - }); - - let matching_list = raw.matching_list.unwrap_or_default(); - err.into_result(Self { - policy: policy.unwrap(), - matching_list, - }) - } -} - /// Settings for `Relayer` #[derive(Debug, AsRef, AsMut, Deref, DerefMut)] pub struct RelayerSettings { @@ -173,48 +56,38 @@ pub struct RelayerSettings { pub allow_local_checkpoint_syncers: bool, } -#[derive(Debug, Deserialize, AsMut)] -#[serde(rename_all = "camelCase")] -pub struct DeprecatedRawRelayerSettings { - #[serde(flatten)] - #[as_mut] - base: DeprecatedRawSettings, - /// Database path (path on the fs) - db: Option, - // Comma separated list of chains to relay between. - relaychains: Option, - // Comma separated list of origin chains. - #[deprecated(note = "Use `relaychains` instead")] - originchainname: Option, - // Comma separated list of destination chains. - #[deprecated(note = "Use `relaychains` instead")] - destinationchainnames: Option, - /// The gas payment enforcement configuration as JSON. Expects an ordered array of `GasPaymentEnforcementConfig`. - gaspaymentenforcement: Option, - /// This is optional. If no whitelist is provided ALL messages will be considered on the - /// whitelist. - whitelist: Option, - /// This is optional. If no blacklist is provided ALL will be considered to not be on - /// the blacklist. - blacklist: Option, - /// This is optional. If not specified, any amount of gas will be valid, otherwise this - /// is the max allowed gas in wei to relay a transaction. - transactiongaslimit: Option, - // TODO: this should be a list of chain names to be consistent - /// Comma separated List of domain ids to skip applying the transaction gas limit to. - skiptransactiongaslimitfor: Option, - /// If true, allows local storage based checkpoint syncers. - /// Not intended for production use. Defaults to false. - #[serde(default)] - allowlocalcheckpointsyncers: bool, +/// Config for gas payment enforcement +#[derive(Debug, Clone, Default)] +pub struct GasPaymentEnforcementConf { + /// The gas payment enforcement policy + pub policy: GasPaymentEnforcementPolicy, + /// An optional matching list, any message that matches will use this + /// policy. By default all messages will match. + pub matching_list: MatchingList, } -impl_loadable_from_settings!(Relayer, DeprecatedRawRelayerSettings -> RelayerSettings); +/// Config for a GasPaymentEnforcementPolicy +#[derive(Debug, Clone, Default)] +pub enum GasPaymentEnforcementPolicy { + /// No requirement - all messages are processed regardless of gas payment + #[default] + None, + /// Messages that have paid a minimum amount will be processed + Minimum { payment: U256 }, + /// The required amount of gas on the foreign chain has been paid according + /// to on-chain fee quoting. + OnChainFeeQuoting { + gas_fraction_numerator: u64, + gas_fraction_denominator: u64, + }, +} #[derive(Debug, Deserialize)] #[serde(transparent)] struct RawRelayerSettings(Value); +impl_loadable_from_settings!(Relayer, RawRelayerSettings -> RelayerSettings); + impl FromRawConf for RelayerSettings { fn from_config_filtered( raw: RawRelayerSettings, @@ -256,7 +129,7 @@ impl FromRawConf for RelayerSettings { }) => serde_json::from_str::(policy_str) .context("Expected JSON string") .take_err(&mut err, || cwp.clone()) - .map(|v| (cwp, v)), + .map(|v| (cwp, recase_json_value(v, Case::Flat))), Some(ValueParser { val: value @ Value::Array(_), cwp, @@ -287,7 +160,7 @@ impl FromRawConf for RelayerSettings { .get_opt_key("gasFraction") .parse_string() .map(|v| v.replace(' ', "")) - .unwrap_or_else(|| default_gasfraction().to_owned()); + .unwrap_or_else(|| "1/2".to_owned()); let (numerator, denominator) = gas_fraction .split_once('/') .ok_or_else(|| eyre!("Invalid `gas_fraction` for OnChainFeeQuoting gas payment enforcement policy; expected `numerator / denominator`")) @@ -394,7 +267,8 @@ fn parse_matching_list(p: ValueParser) -> ConfigResult { cwp, } => serde_json::from_str::(matching_list_str) .context("Expected JSON string") - .take_err(&mut err, || cwp.clone()), + .take_err(&mut err, || cwp.clone()) + .map(|v| recase_json_value(v, Case::Flat)), ValueParser { val: value @ Value::Array(_), .. @@ -413,210 +287,3 @@ fn parse_matching_list(p: ValueParser) -> ConfigResult { err.into_result(ml) } - -impl FromRawConf for RelayerSettings { - fn from_config_filtered( - raw: DeprecatedRawRelayerSettings, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let gas_payment_enforcement = raw - .gaspaymentenforcement - .and_then(|j| { - serde_json::from_str::>(&j) - .take_err(&mut err, || cwp + "gaspaymentenforcement") - }) - .map(|rv| { - let cwp = cwp + "gaspaymentenforcement"; - rv.into_iter() - .enumerate() - .filter_map(|(i, r)| { - r.parse_config(&cwp.join(i.to_string())) - .take_config_err(&mut err) - }) - .collect() - }) - .unwrap_or_else(|| vec![Default::default()]); - - let whitelist = raw - .whitelist - .and_then(|j| { - serde_json::from_str::(&j).take_err(&mut err, || cwp + "whitelist") - }) - .unwrap_or_default(); - - let blacklist = raw - .blacklist - .and_then(|j| { - serde_json::from_str::(&j).take_err(&mut err, || cwp + "blacklist") - }) - .unwrap_or_default(); - - let transaction_gas_limit = raw.transactiongaslimit.and_then(|r| { - r.try_into() - .take_err(&mut err, || cwp + "transactiongaslimit") - }); - - let skip_transaction_gas_limit_for = raw - .skiptransactiongaslimitfor - .and_then(|r| { - r.split(',') - .map(str::parse) - .collect::>() - .context("Error parsing domain id") - .take_err(&mut err, || cwp + "skiptransactiongaslimitfor") - }) - .unwrap_or_default(); - - let mut origin_chain_names = { - #[allow(deprecated)] - raw.originchainname - } - .map(parse_chains); - - if origin_chain_names.is_some() { - warn!( - path = (cwp + "originchainname").json_name(), - "`originchainname` is deprecated, use `relaychains` instead" - ); - } - - let mut destination_chain_names = { - #[allow(deprecated)] - raw.destinationchainnames - } - .map(parse_chains); - - if destination_chain_names.is_some() { - warn!( - path = (cwp + "destinationchainnames").json_name(), - "`destinationchainnames` is deprecated, use `relaychains` instead" - ); - } - - if let Some(relay_chain_names) = raw.relaychains.map(parse_chains) { - if origin_chain_names.is_some() { - err.push( - cwp + "originchainname", - eyre!("Cannot use `relaychains` and `originchainname` at the same time"), - ); - } - if destination_chain_names.is_some() { - err.push( - cwp + "destinationchainnames", - eyre!("Cannot use `relaychains` and `destinationchainnames` at the same time"), - ); - } - - if relay_chain_names.len() < 2 { - err.push( - cwp + "relaychains", - eyre!( - "The relayer must be configured with at least two chains to relay between" - ), - ) - } - origin_chain_names = Some(relay_chain_names.clone()); - destination_chain_names = Some(relay_chain_names); - } else if origin_chain_names.is_none() && destination_chain_names.is_none() { - err.push( - cwp + "relaychains", - eyre!("The relayer must be configured with at least two chains to relay between"), - ); - } else if origin_chain_names.is_none() { - err.push( - cwp + "originchainname", - eyre!("The relayer must be configured with an origin chain (alternatively use `relaychains`)"), - ); - } else if destination_chain_names.is_none() { - err.push( - cwp + "destinationchainnames", - eyre!("The relayer must be configured with at least one destination chain (alternatively use `relaychains`)"), - ); - } - - let db = raw - .db - .and_then(|r| r.parse().take_err(&mut err, || cwp + "db")) - .unwrap_or_else(|| std::env::current_dir().unwrap().join("hyperlane_db")); - - let (Some(origin_chain_names), Some(destination_chain_names)) = - (origin_chain_names, destination_chain_names) - else { return Err(err) }; - - let chain_filter = origin_chain_names - .iter() - .chain(&destination_chain_names) - .map(String::as_str) - .collect(); - - let base = raw - .base - .parse_config_with_filter::(cwp, Some(&chain_filter)) - .take_config_err(&mut err); - - let origin_chains = base - .as_ref() - .map(|base| { - origin_chain_names - .iter() - .filter_map(|origin| { - base.lookup_domain(origin) - .context("Missing configuration for an origin chain") - .take_err(&mut err, || cwp + "chains" + origin) - }) - .collect() - }) - .unwrap_or_default(); - - // validate all destination chains are present and get their HyperlaneDomain. - let destination_chains: HashSet<_> = base - .as_ref() - .map(|base| { - destination_chain_names - .iter() - .filter_map(|destination| { - base.lookup_domain(destination) - .context("Missing configuration for a destination chain") - .take_err(&mut err, || cwp + "chains" + destination) - }) - .collect() - }) - .unwrap_or_default(); - - if let Some(base) = &base { - for domain in &destination_chains { - base.chain_setup(domain) - .unwrap() - .signer - .as_ref() - .ok_or_else(|| eyre!("Signer is required for destination chains")) - .take_err(&mut err, || cwp + "chains" + domain.name() + "signer"); - } - } - - cfg_unwrap_all!(cwp, err: [base]); - err.into_result(Self { - base, - db, - origin_chains, - destination_chains, - gas_payment_enforcement, - whitelist, - blacklist, - transaction_gas_limit, - skip_transaction_gas_limit_for, - allow_local_checkpoint_syncers: raw.allowlocalcheckpointsyncers, - }) - } -} - -fn default_gasfraction() -> String { - "1/2".into() -} - -fn parse_chains(chains_str: String) -> Vec { - chains_str.split(',').map(str::to_ascii_lowercase).collect() -} diff --git a/rust/agents/scraper/src/settings.rs b/rust/agents/scraper/src/settings.rs index 360c8f1fe7..b4bdfbb4d5 100644 --- a/rust/agents/scraper/src/settings.rs +++ b/rust/agents/scraper/src/settings.rs @@ -7,17 +7,15 @@ use std::{collections::HashSet, default::Default}; use derive_more::{AsMut, AsRef, Deref, DerefMut}; -use eyre::{eyre, Context}; +use eyre::Context; use hyperlane_base::{ impl_loadable_from_settings, settings::{ - deprecated_parser::DeprecatedRawSettings, parser::{RawAgentConf, ValueParser}, Settings, }, }; use hyperlane_core::{cfg_unwrap_all, config::*, HyperlaneDomain}; -use itertools::Itertools; use serde::Deserialize; use serde_json::Value; @@ -34,25 +32,12 @@ pub struct ScraperSettings { pub chains_to_scrape: Vec, } -/// Raw settings for `Scraper` -#[derive(Debug, Deserialize, AsMut)] -#[serde(rename_all = "camelCase")] -pub struct DeprecatedRawScraperSettings { - #[serde(flatten, default)] - #[as_mut] - base: DeprecatedRawSettings, - /// Database connection string - db: Option, - /// Comma separated list of chains to scrape - chainstoscrape: Option, -} - -impl_loadable_from_settings!(Scraper, DeprecatedRawScraperSettings -> ScraperSettings); - #[derive(Debug, Deserialize)] #[serde(transparent)] struct RawScraperSettings(Value); +impl_loadable_from_settings!(Scraper, RawScraperSettings -> ScraperSettings); + impl FromRawConf for ScraperSettings { fn from_config_filtered( raw: RawScraperSettings, @@ -107,53 +92,3 @@ impl FromRawConf for ScraperSettings { }) } } - -impl FromRawConf for ScraperSettings { - fn from_config_filtered( - raw: DeprecatedRawScraperSettings, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let db = raw - .db - .ok_or_else(|| eyre!("Missing `db` connection string")) - .take_err(&mut err, || cwp + "db"); - - let Some(chains_to_scrape) = raw - .chainstoscrape - .ok_or_else(|| eyre!("Missing `chainstoscrape` list")) - .take_err(&mut err, || cwp + "chainstoscrape") - .map(|s| s.split(',').map(str::to_ascii_lowercase).collect::>()) - else { return Err(err) }; - - let base = raw - .base - .parse_config_with_filter::( - cwp, - Some(&chains_to_scrape.iter().map(String::as_str).collect()), - ) - .take_config_err(&mut err); - - let chains_to_scrape = base - .as_ref() - .map(|base| { - chains_to_scrape - .iter() - .filter_map(|chain| { - base.lookup_domain(chain) - .context("Missing configuration for a chain in `chainstoscrape`") - .take_err(&mut err, || cwp + "chains" + chain) - }) - .collect_vec() - }) - .unwrap_or_default(); - - err.into_result(Self { - base: base.unwrap(), - db: db.unwrap(), - chains_to_scrape, - }) - } -} diff --git a/rust/agents/validator/src/settings.rs b/rust/agents/validator/src/settings.rs index ea5b1f7893..4c2c673b2a 100644 --- a/rust/agents/validator/src/settings.rs +++ b/rust/agents/validator/src/settings.rs @@ -11,9 +11,6 @@ use eyre::{eyre, Context}; use hyperlane_base::{ impl_loadable_from_settings, settings::{ - deprecated_parser::{ - DeprecatedRawCheckpointSyncerConf, DeprecatedRawSettings, DeprecatedRawSignerConf, - }, parser::{RawAgentConf, RawAgentSignerConf, ValueParser}, CheckpointSyncerConf, Settings, SignerConf, }, @@ -45,34 +42,12 @@ pub struct ValidatorSettings { pub interval: Duration, } -/// Raw settings for `Validator` -#[derive(Debug, Deserialize, AsMut)] -#[serde(rename_all = "camelCase")] -pub struct DeprecatedRawValidatorSettings { - #[serde(flatten, default)] - #[as_mut] - base: DeprecatedRawSettings, - /// Database path (path on the fs) - db: Option, - // Name of the chain to validate message on - originchainname: Option, - /// The validator attestation signer - #[serde(default)] - validator: DeprecatedRawSignerConf, - /// The checkpoint syncer configuration - checkpointsyncer: Option, - /// The reorg_period in blocks - reorgperiod: Option, - /// How frequently to check for new checkpoints - interval: Option, -} - -impl_loadable_from_settings!(Validator, DeprecatedRawValidatorSettings -> ValidatorSettings); - #[derive(Debug, Deserialize)] #[serde(transparent)] struct RawValidatorSettings(Value); +impl_loadable_from_settings!(Validator, RawValidatorSettings -> ValidatorSettings); + impl FromRawConf for ValidatorSettings { fn from_config_filtered( raw: RawValidatorSettings, @@ -151,6 +126,14 @@ impl FromRawConf for ValidatorSettings { cfg_unwrap_all!(cwp, err: [base, origin_chain, validator, checkpoint_syncer]); + let mut base: Settings = base; + // If the origin chain is an EVM chain, then we can use the validator as the signer if needed. + if origin_chain.domain_protocol() == HyperlaneDomainProtocol::Ethereum { + if let Some(origin) = base.chains.get_mut(origin_chain.name()) { + origin.signer.get_or_insert_with(|| validator.clone()); + } + } + err.into_result(Self { base, db, @@ -210,93 +193,3 @@ fn parse_checkpoint_syncer(syncer: ValueParser) -> ConfigResult Err(err), } } - -impl FromRawConf for ValidatorSettings { - fn from_config_filtered( - raw: DeprecatedRawValidatorSettings, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let validator = raw - .validator - .parse_config::(&cwp.join("validator")) - .take_config_err(&mut err); - - let checkpoint_syncer = raw - .checkpointsyncer - .ok_or_else(|| eyre!("Missing `checkpointsyncer`")) - .take_err(&mut err, || cwp + "checkpointsyncer") - .and_then(|r| { - r.parse_config(&cwp.join("checkpointsyncer")) - .take_config_err(&mut err) - }); - - let reorg_period = raw - .reorgperiod - .ok_or_else(|| eyre!("Missing `reorgperiod`")) - .take_err(&mut err, || cwp + "reorgperiod") - .and_then(|r| r.try_into().take_err(&mut err, || cwp + "reorgperiod")); - - let interval = raw - .interval - .and_then(|r| { - r.try_into() - .map(Duration::from_secs) - .take_err(&mut err, || cwp + "interval") - }) - .unwrap_or(Duration::from_secs(5)); - - let Some(origin_chain_name) = raw - .originchainname - .ok_or_else(|| eyre!("Missing `originchainname`")) - .take_err(&mut err, || cwp + "originchainname") - .map(|s| s.to_ascii_lowercase()) - else { return Err(err) }; - - let db = raw - .db - .and_then(|r| r.parse().take_err(&mut err, || cwp + "db")) - .unwrap_or_else(|| { - std::env::current_dir() - .unwrap() - .join(format!("validator_db_{origin_chain_name}")) - }); - - let base = raw - .base - .parse_config_with_filter::( - cwp, - Some(&[origin_chain_name.as_ref()].into_iter().collect()), - ) - .take_config_err(&mut err); - - let origin_chain = base.as_ref().and_then(|base| { - base.lookup_domain(&origin_chain_name) - .context("Missing configuration for the origin chain") - .take_err(&mut err, || cwp + "chains" + &origin_chain_name) - }); - - cfg_unwrap_all!(cwp, err: [base, origin_chain, validator, checkpoint_syncer, reorg_period]); - let mut base = base; - - if origin_chain.domain_protocol() == HyperlaneDomainProtocol::Ethereum { - // if an EVM chain we can assume the chain signer is the validator signer when not - // specified - if let Some(chain) = base.chains.get_mut(origin_chain.name()) { - chain.signer.get_or_insert_with(|| validator.clone()); - } - } - - err.into_result(Self { - base, - db, - origin_chain, - validator, - checkpoint_syncer, - reorg_period, - interval, - }) - } -} diff --git a/rust/chains/hyperlane-ethereum/src/config.rs b/rust/chains/hyperlane-ethereum/src/config.rs index fc494784e9..96500385bb 100644 --- a/rust/chains/hyperlane-ethereum/src/config.rs +++ b/rust/chains/hyperlane-ethereum/src/config.rs @@ -1,5 +1,3 @@ -use hyperlane_core::config::*; -use serde::Deserialize; use url::Url; /// Ethereum connection configuration @@ -26,96 +24,3 @@ pub enum ConnectionConf { url: Url, }, } - -/// Ethereum connection configuration -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RawConnectionConf { - /// The type of connection to use - #[serde(rename = "type")] - connection_type: Option, - /// A single url to connect to - url: Option, - /// A comma separated list of urls to connect to - urls: Option, -} - -/// Error type when parsing a connection configuration. -#[derive(Debug, thiserror::Error)] -pub enum ConnectionConfError { - /// Unknown connection type was specified - #[error("Unsupported connection type '{0}'")] - UnsupportedConnectionType(String), - /// The url was not specified - #[error("Missing `url` for connection configuration")] - MissingConnectionUrl, - /// The urls were not specified - #[error("Missing `urls` for connection configuration")] - MissingConnectionUrls, - /// The could not be parsed - #[error("Invalid `url` for connection configuration: `{0}` ({1})")] - InvalidConnectionUrl(String, url::ParseError), - /// One of the urls could not be parsed - #[error("Invalid `urls` list for connection configuration: `{0}` ({1})")] - InvalidConnectionUrls(String, url::ParseError), - /// The url was empty - #[error("The `url` value is empty")] - EmptyUrl, - /// The urls were empty - #[error("The `urls` value is empty")] - EmptyUrls, -} - -impl FromRawConf for ConnectionConf { - fn from_config_filtered( - raw: RawConnectionConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - use ConnectionConfError::*; - - let connection_type = raw.connection_type.as_deref().unwrap_or("http"); - - let urls = (|| -> ConfigResult> { - raw.urls - .as_ref() - .ok_or(MissingConnectionUrls) - .into_config_result(|| cwp + "urls")? - .split(',') - .map(|s| s.parse()) - .collect::, _>>() - .map_err(|e| InvalidConnectionUrls(raw.urls.clone().unwrap(), e)) - .into_config_result(|| cwp + "urls") - })(); - - let url = (|| -> ConfigResult { - raw.url - .as_ref() - .ok_or(MissingConnectionUrl) - .into_config_result(|| cwp + "url")? - .parse() - .map_err(|e| InvalidConnectionUrl(raw.url.clone().unwrap(), e)) - .into_config_result(|| cwp + "url") - })(); - - macro_rules! make_with_urls { - ($variant:ident) => { - if let Ok(urls) = urls { - Ok(Self::$variant { urls }) - } else if let Ok(url) = url { - Ok(Self::$variant { urls: vec![url] }) - } else { - Err(urls.unwrap_err()) - } - }; - } - - match connection_type { - "httpQuorum" => make_with_urls!(HttpQuorum), - "httpFallback" => make_with_urls!(HttpFallback), - "http" => Ok(Self::Http { url: url? }), - "ws" => Ok(Self::Ws { url: url? }), - t => Err(UnsupportedConnectionType(t.into())).into_config_result(|| cwp.join("type")), - } - } -} diff --git a/rust/chains/hyperlane-fuel/src/trait_builder.rs b/rust/chains/hyperlane-fuel/src/trait_builder.rs index a8a107e870..b056a17ee2 100644 --- a/rust/chains/hyperlane-fuel/src/trait_builder.rs +++ b/rust/chains/hyperlane-fuel/src/trait_builder.rs @@ -1,5 +1,5 @@ use fuels::{client::FuelClient, prelude::Provider}; -use hyperlane_core::{config::*, ChainCommunicationError, ChainResult}; +use hyperlane_core::{ChainCommunicationError, ChainResult}; use url::Url; /// Fuel connection configuration @@ -9,12 +9,6 @@ pub struct ConnectionConf { pub url: Url, } -/// Raw fuel connection configuration used for better deserialization errors. -#[derive(Debug, serde::Deserialize)] -pub struct DeprecatedRawConnectionConf { - url: Option, -} - /// An error type when parsing a connection configuration. #[derive(thiserror::Error, Debug)] pub enum ConnectionConfError { @@ -26,27 +20,6 @@ pub enum ConnectionConfError { InvalidConnectionUrl(String, url::ParseError), } -impl FromRawConf for ConnectionConf { - fn from_config_filtered( - raw: DeprecatedRawConnectionConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - use ConnectionConfError::*; - match raw { - DeprecatedRawConnectionConf { url: Some(url) } => Ok(Self { - url: url - .parse() - .map_err(|e| InvalidConnectionUrl(url, e)) - .into_config_result(|| cwp.join("url"))?, - }), - DeprecatedRawConnectionConf { url: None } => { - Err(MissingConnectionUrl).into_config_result(|| cwp.join("url")) - } - } - } -} - #[derive(thiserror::Error, Debug)] #[error(transparent)] struct FuelNewConnectionError(#[from] anyhow::Error); diff --git a/rust/chains/hyperlane-sealevel/src/trait_builder.rs b/rust/chains/hyperlane-sealevel/src/trait_builder.rs index d505eab8b2..8b14b868e0 100644 --- a/rust/chains/hyperlane-sealevel/src/trait_builder.rs +++ b/rust/chains/hyperlane-sealevel/src/trait_builder.rs @@ -1,7 +1,4 @@ -use hyperlane_core::{ - config::{ConfigErrResultExt, ConfigPath, ConfigResult, FromRawConf}, - ChainCommunicationError, -}; +use hyperlane_core::ChainCommunicationError; use url::Url; /// Sealevel connection configuration @@ -11,12 +8,6 @@ pub struct ConnectionConf { pub url: Url, } -/// Raw Sealevel connection configuration used for better deserialization errors. -#[derive(Debug, serde::Deserialize)] -pub struct DeprecatedRawConnectionConf { - url: Option, -} - /// An error type when parsing a connection configuration. #[derive(thiserror::Error, Debug)] pub enum ConnectionConfError { @@ -28,27 +19,6 @@ pub enum ConnectionConfError { InvalidConnectionUrl(String, url::ParseError), } -impl FromRawConf for ConnectionConf { - fn from_config_filtered( - raw: DeprecatedRawConnectionConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - use ConnectionConfError::*; - match raw { - DeprecatedRawConnectionConf { url: Some(url) } => Ok(Self { - url: url - .parse() - .map_err(|e| InvalidConnectionUrl(url, e)) - .into_config_result(|| cwp.join("url"))?, - }), - DeprecatedRawConnectionConf { url: None } => { - Err(MissingConnectionUrl).into_config_result(|| cwp.join("url")) - } - } - } -} - #[derive(thiserror::Error, Debug)] #[error(transparent)] struct SealevelNewConnectionError(#[from] anyhow::Error); diff --git a/rust/config/test_sealevel_config.json b/rust/config/test_sealevel_config.json index af8d8d48b9..5c127dcc3d 100644 --- a/rust/config/test_sealevel_config.json +++ b/rust/config/test_sealevel_config.json @@ -2,18 +2,21 @@ "chains": { "sealeveltest1": { "name": "sealeveltest1", - "domain": 13375, - "addresses": { - "mailbox": "692KZJaoe2KRcD6uhCQDLLXnLNA5ZLnfvdqjE4aX9iu1", - "interchainGasPaymaster": "DrFtxirPPsfdY4HQiNZj2A9o4Ux7JaL3gELANgAoihhp", - "validatorAnnounce": "DH43ae1LwemXAboWwSh8zc9pG8j72gKUEXNi57w8fEnn" - }, + "chainId": 13375, + "domainId": 13375, + "mailbox": "692KZJaoe2KRcD6uhCQDLLXnLNA5ZLnfvdqjE4aX9iu1", + "interchainGasPaymaster": "DrFtxirPPsfdY4HQiNZj2A9o4Ux7JaL3gELANgAoihhp", + "validatorAnnounce": "DH43ae1LwemXAboWwSh8zc9pG8j72gKUEXNi57w8fEnn", "protocol": "sealevel", - "finalityBlocks": 0, - "connection": { - "type": "http", - "url": "http://localhost:8899" + "blocks": { + "reorgPeriod": 0, + "confirmations": 0 }, + "rpcUrls": [ + { + "http": "http://localhost:8899" + } + ], "index": { "from": 1, "mode": "sequence" @@ -21,18 +24,21 @@ }, "sealeveltest2": { "name": "sealeveltest2", - "domain": 13376, - "addresses": { - "mailbox": "9tCUWNjpqcf3NUSrtp7vquYVCwbEByvLjZUrhG5dgvhj", - "interchainGasPaymaster": "G5rGigZBL8NmxCaukK2CAKr9Jq4SUfAhsjzeri7GUraK", - "validatorAnnounce": "3Uo5j2Bti9aZtrDqJmAyuwiFaJFPFoNL5yxTpVCNcUhb" - }, + "chainId": 13376, + "domainId": 13376, + "mailbox": "9tCUWNjpqcf3NUSrtp7vquYVCwbEByvLjZUrhG5dgvhj", + "interchainGasPaymaster": "G5rGigZBL8NmxCaukK2CAKr9Jq4SUfAhsjzeri7GUraK", + "validatorAnnounce": "3Uo5j2Bti9aZtrDqJmAyuwiFaJFPFoNL5yxTpVCNcUhb", "protocol": "sealevel", - "finalityBlocks": 0, - "connection": { - "type": "http", - "url": "http://localhost:8899" + "blocks": { + "reorgPeriod": 0, + "confirmations": 0 }, + "rpcUrls": [ + { + "http": "http://localhost:8899" + } + ], "index": { "from": 1, "mode": "sequence" diff --git a/rust/helm/agent-common/templates/_helpers.tpl b/rust/helm/agent-common/templates/_helpers.tpl index bf80d907f7..95b93224a8 100644 --- a/rust/helm/agent-common/templates/_helpers.tpl +++ b/rust/helm/agent-common/templates/_helpers.tpl @@ -73,30 +73,30 @@ The name of the ClusterSecretStore/SecretStore {{/* Recursively converts a config object into environment variables than can -be parsed by rust. For example, a config of { foo: { bar: { baz: 420 }, boo: 421 } } will -be: HYP_FOO_BAR_BAZ=420 and HYP_FOO_BOO=421 +be parsed by rust. For example, a config of { foo: { bar: { baz: 420 }, booGo: 421 } } will +be: HYP_FOO_BAR_BAZ=420 and HYP_FOO_BOOGO=421 Env vars can be formatted in FOO="BAR" format if .format is "dot_env", FOO: "BAR" format if .format is "config_map", or otherwise they will be formatted as spec YAML-friendly environment variables */}} {{- define "agent-common.config-env-vars" -}} -{{- range $key, $value := .config }} -{{- $key_name := printf "%s%s" (default "" $.key_name_prefix) $key }} -{{- if typeIs "map[string]interface {}" $value }} -{{- include "agent-common.config-env-vars" (dict "config" $value "agent_name" $.agent_name "format" $.format "key_name_prefix" (printf "%s_" $key_name)) }} +{{- range $key_or_idx, $value := .config }} +{{- $key_name := printf "%s%v" (default "" $.key_name_prefix) $key_or_idx }} +{{- if or (typeIs "map[string]interface {}" $value) (typeIs "[]interface {}" $value) }} +{{- include "agent-common.config-env-vars" (dict "config" $value "format" $.format "key_name_prefix" (printf "%s_" $key_name)) }} {{- else }} -{{- include "agent-common.config-env-var" (dict "agent_name" $.agent_name "key" $key_name "value" $value "format" $.format ) }} +{{- include "agent-common.config-env-var" (dict "key" $key_name "value" $value "format" $.format ) }} {{- end }} {{- end }} {{- end }} {{- define "agent-common.config-env-var" }} {{- if (eq .format "dot_env") }} -HYP_{{ .agent_name | upper }}_{{ .key | upper }}={{ .value | quote }} +HYP_{{ .key | upper }}={{ .value | quote }} {{- else if (eq .format "config_map") }} -HYP_{{ .agent_name | upper }}_{{ .key | upper }}: {{ .value | quote }} +HYP_{{ .key | upper }}: {{ .value | quote }} {{- else }} -- name: HYP_{{ .agent_name | upper }}_{{ .key | upper }} +- name: HYP_{{ .key | upper }} value: {{ .value | quote }} {{- end }} {{- end }} diff --git a/rust/helm/hyperlane-agent/README.md b/rust/helm/hyperlane-agent/README.md deleted file mode 100644 index 9894658ff0..0000000000 --- a/rust/helm/hyperlane-agent/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Hyperlane-Agent Helm Chart - -![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square) - -A Helm Chart that encapsulates the deployment of the Hyperlane Rust Agent(s). It is currently designed to be deployed against a Google Kubernetes Engine cluster, but specification of another PVC Storage Class should be sufficient to make it compatible with other cloud providers. - -Additional documentation is present in comments in `yalues.yaml`. - -## Values - -| Key | Type | Default | Description | -| -------------------------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| affinity | object | `{}` | | -| fullnameOverride | string | `""` | | -| image.pullPolicy | string | `"Always"` | | -| image.repository | string | `"gcr.io/clabs-optics/optics-agent"` | Main repository for Hyperlane Agent binaries, provided by cLabs | -| image.tag | string | `"latest"` | Overrides the image tag whose default is the chart appVersion. | -| imagePullSecrets | list | `[]` | | -| nameOverride | string | `""` | | -| nodeSelector | object | `{}` | | -| hyperlane | object | `{"outboxChain":{"address":null,"connectionType":null,"connectionUrl":null,"domain":null,"name":"goerli","protocol":null},enabled":false,"messageInterval":null,"signers":[{"key":"","name":"goerli"},{"key":"","name":"alfajores"}]},"processor":{"enabled":false,"pollingInterval":null,"signers":[{"key":"","name":"goerli"},{"key":"","name":"alfajores"}]},"relayer":{"enabled":false,"pollingInterval":null,"signers":[{"key":"","name":"goerli"},{"key":"","name":"alfajores"}]},"inboxChains":[{"address":null,"connectionType":null,"connectionUrl":null,"domain":null,"name":"alfajores","protocol":null}],"runEnv":"default","validator":{"signer":"","enabled":false,"pollingInterval":null,"signers":[{"key":"","name":"goerli"},{"key":"","name":"alfajores"}],"updatePause":null}}` | Hyperlane Overrides By Default, Hyperlane Agents load the config baked into the Docker Image Pass values here in order to override the values in the config Note: For successful operation, one _must_ pass signer keys as they are not baked into the image for security reasons. | -| hyperlane.outboxChain.address | string | `nil` | The contract address for the home contract | -| hyperlane.outboxChain.connectionUrl | string | `nil` | Connection string pointing to an RPC endpoint for the home chain | -| hyperlane.outboxChain.domain | string | `nil` | The hard-coded domain corresponding to this blockchain | -| hyperlane.outboxChain.protocol | string | `nil` | RPC Style | -| hyperlane.relayer.enabled | bool | `false` | Enables or disables the relayer | -| hyperlane.inboxChains | list | `[{"address":null,"connectionType":null,"connectionUrl":null,"domain":null,"name":"alfajores","protocol":null}]` | Replica chain overrides, a sequence | -| hyperlane.inboxChains[0].address | string | `nil` | The contract address for the replica contract | -| hyperlane.inboxChains[0].connectionUrl | string | `nil` | Connection string pointing to an RPC endpoint for the replica chain | -| hyperlane.validator.signer | string | `""` | Specialized key used by validator and watcher used to sign attestations, separate from validator.keys | -| hyperlane.validator.enabled | bool | `false` | Enables or disables the validator | -| hyperlane.validator.pollingInterval | string | `nil` | How long to wait between checking for updates | -| hyperlane.validator.signers | list | `[{"key":"","name":"goerli"},{"key":"","name":"alfajores"}]` | Trnsaction Signing keys for home and replica(s) | -| podAnnotations | object | `{}` | | -| podSecurityContext | object | `{}` | | -| replicaCount | int | `1` | | -| resources | object | `{}` | | -| securityContext | object | `{}` | | -| tolerations | list | `[]` | | -| volumeStorageClass | string | `"standard"` | Default to standard storageclass provided by GKE | - ---- - -Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) diff --git a/rust/helm/hyperlane-agent/templates/configmap.yaml b/rust/helm/hyperlane-agent/templates/configmap.yaml index 6e6123b749..2ce9ae827e 100644 --- a/rust/helm/hyperlane-agent/templates/configmap.yaml +++ b/rust/helm/hyperlane-agent/templates/configmap.yaml @@ -7,10 +7,10 @@ metadata: data: ONELINE_BACKTRACES: "true" RUST_BACKTRACE: {{ .Values.hyperlane.rustBacktrace }} - HYP_BASE_DB: {{ .Values.hyperlane.dbPath }} - HYP_BASE_TRACING_FMT: {{ .Values.hyperlane.tracing.format }} - HYP_BASE_TRACING_LEVEL: {{ .Values.hyperlane.tracing.level }} + HYP_DB: {{ .Values.hyperlane.dbPath }} + HYP_LOG_FORMAT: {{ .Values.hyperlane.tracing.format }} + HYP_LOG_LEVEL: {{ .Values.hyperlane.tracing.level }} {{- range .Values.hyperlane.chains }} -{{- include "agent-common.config-env-vars" (dict "config" . "agent_name" "base" "key_name_prefix" (printf "CHAINS_%s_" (.name | upper)) "format" "config_map") | indent 2 }} +{{- include "agent-common.config-env-vars" (dict "config" . "key_name_prefix" (printf "chains_%s_" .name) "format" "config_map") | indent 2 }} {{- end }} - HYP_BASE_METRICS: {{ .Values.hyperlane.metrics.port | quote }} + HYP_METRICSPORT: {{ .Values.hyperlane.metrics.port | quote }} diff --git a/rust/helm/hyperlane-agent/templates/external-secret.yaml b/rust/helm/hyperlane-agent/templates/external-secret.yaml index 1b6ac58498..023747a915 100644 --- a/rust/helm/hyperlane-agent/templates/external-secret.yaml +++ b/rust/helm/hyperlane-agent/templates/external-secret.yaml @@ -27,11 +27,7 @@ spec: */}} {{- range .Values.hyperlane.chains }} {{- if not .disabled }} - {{- if or (eq .connection.type "httpQuorum") (eq .connection.type "httpFallback") }} - HYP_BASE_CHAINS_{{ .name | upper }}_CONNECTION_URLS: {{ printf "'{{ .%s_rpcs | fromJson | join \",\" }}'" .name }} - {{- else }} - HYP_BASE_CHAINS_{{ .name | upper }}_CONNECTION_URL: {{ printf "'{{ .%s_rpc | toString }}'" .name }} - {{- end }} + HYP_BASE_CHAINS_{{ .name | upper }}_CUSTOMRPCURLS: {{ printf "'{{ .%s_rpcs | fromJson | join \",\" }}'" .name }} {{- end }} {{- end }} data: @@ -41,14 +37,8 @@ spec: */}} {{- range .Values.hyperlane.chains }} {{- if not .disabled }} - {{- if or (eq .connection.type "httpQuorum") (eq .connection.type "httpFallback") }} - secretKey: {{ printf "%s_rpcs" .name }} remoteRef: key: {{ printf "%s-rpc-endpoints-%s" $.Values.hyperlane.runEnv .name }} - {{- else }} - - secretKey: {{ printf "%s_rpc" .name }} - remoteRef: - key: {{ printf "%s-rpc-endpoint-%s" $.Values.hyperlane.runEnv .name }} - {{- end }} {{- end }} {{- end }} diff --git a/rust/helm/hyperlane-agent/templates/relayer-external-secret.yaml b/rust/helm/hyperlane-agent/templates/relayer-external-secret.yaml index 182ad70457..da26afd2e4 100644 --- a/rust/helm/hyperlane-agent/templates/relayer-external-secret.yaml +++ b/rust/helm/hyperlane-agent/templates/relayer-external-secret.yaml @@ -23,13 +23,13 @@ spec: data: {{- range .Values.hyperlane.relayerChains }} {{- if eq .signer.type "hexKey" }} - HYP_BASE_CHAINS_{{ .name | upper }}_SIGNER_KEY: {{ printf "'{{ .%s_signer_key | toString }}'" .name }} + HYP_CHAINS_{{ .name | upper }}_SIGNER_KEY: {{ printf "'{{ .%s_signer_key | toString }}'" .name }} {{- end }} - {{- end }} - {{- if .Values.hyperlane.relayer.aws }} + {{- if and (eq .signer.type "aws") $.Values.hyperlane.relayer.aws }} AWS_ACCESS_KEY_ID: {{ print "'{{ .aws_access_key_id | toString }}'" }} AWS_SECRET_ACCESS_KEY: {{ print "'{{ .aws_secret_access_key | toString }}'" }} {{- end }} + {{- end }} data: {{- range .Values.hyperlane.relayerChains }} {{- if eq .signer.type "hexKey" }} diff --git a/rust/helm/hyperlane-agent/templates/relayer-statefulset.yaml b/rust/helm/hyperlane-agent/templates/relayer-statefulset.yaml index 8d8e8d50de..4160e0c25b 100644 --- a/rust/helm/hyperlane-agent/templates/relayer-statefulset.yaml +++ b/rust/helm/hyperlane-agent/templates/relayer-statefulset.yaml @@ -55,14 +55,7 @@ spec: - secretRef: name: {{ include "agent-common.fullname" . }}-relayer-secret env: -{{- include "agent-common.config-env-vars" (dict "config" .Values.hyperlane.relayer.config "agent_name" "relayer") | indent 10 }} -{{- $relayerChainNames := list }} - {{- range .Values.hyperlane.relayerChains }} -{{- include "agent-common.config-env-vars" (dict "config" .signer "agent_name" "base" "key_name_prefix" (printf "CHAINS_%s_SIGNER_" (.name | upper))) | indent 10 }} -{{- $relayerChainNames = append $relayerChainNames .name }} - {{- end }} - - name: HYP_BASE_RELAYCHAINS - value: {{ $relayerChainNames | join "," }} + {{- include "agent-common.config-env-vars" (dict "config" .Values.hyperlane.relayer.config) | nindent 10 }} resources: {{- toYaml .Values.hyperlane.relayer.resources | nindent 10 }} volumeMounts: diff --git a/rust/helm/hyperlane-agent/templates/scraper-external-secret.yaml b/rust/helm/hyperlane-agent/templates/scraper-external-secret.yaml index 1ee160eda1..875534dffb 100644 --- a/rust/helm/hyperlane-agent/templates/scraper-external-secret.yaml +++ b/rust/helm/hyperlane-agent/templates/scraper-external-secret.yaml @@ -21,7 +21,7 @@ spec: labels: {{- include "agent-common.labels" . | nindent 10 }} data: - HYP_BASE_DB: {{ print "'{{ .db | toString }}'" }} + HYP_DB: {{ print "'{{ .db | toString }}'" }} data: - secretKey: db remoteRef: diff --git a/rust/helm/hyperlane-agent/templates/scraper-statefulset.yaml b/rust/helm/hyperlane-agent/templates/scraper-statefulset.yaml index 962ae0c662..06326e260c 100644 --- a/rust/helm/hyperlane-agent/templates/scraper-statefulset.yaml +++ b/rust/helm/hyperlane-agent/templates/scraper-statefulset.yaml @@ -55,14 +55,7 @@ spec: - secretRef: name: {{ include "agent-common.fullname" . }}-scraper3-secret env: -{{- $scraperChainNames := list }} -{{- range .Values.hyperlane.chains }} -{{- if not .disabled }} -{{- $scraperChainNames = append $scraperChainNames .name }} -{{- end }} -{{- end }} - - name: HYP_SCRAPER_CHAINSTOSCRAPE - value: {{ $scraperChainNames | join "," }} + {{- include "agent-common.config-env-vars" (dict "config" .Values.hyperlane.scraper.config) | nindent 8 }} resources: {{- toYaml .Values.hyperlane.scraper.resources | nindent 10 }} ports: diff --git a/rust/helm/hyperlane-agent/templates/validator-configmap.yaml b/rust/helm/hyperlane-agent/templates/validator-configmap.yaml index 9d2dae60d0..da452f4fa8 100644 --- a/rust/helm/hyperlane-agent/templates/validator-configmap.yaml +++ b/rust/helm/hyperlane-agent/templates/validator-configmap.yaml @@ -6,10 +6,8 @@ metadata: labels: {{- include "agent-common.labels" . | nindent 4 }} data: -{{ $index := 0 }} -{{- range .Values.hyperlane.validator.configs }} +{{- range $index, $config := .Values.hyperlane.validator.configs }} validator-{{ $index }}.env: | -{{- include "agent-common.config-env-vars" (dict "config" . "agent_name" "validator" "format" "dot_env") | indent 4 }} -{{ $index = add1 $index }} + {{- include "agent-common.config-env-vars" (dict "config" $config "format" "dot_env") | nindent 4 }} {{- end }} {{- end }} diff --git a/rust/helm/hyperlane-agent/templates/validator-external-secret.yaml b/rust/helm/hyperlane-agent/templates/validator-external-secret.yaml index 79e67a7891..c38900483a 100644 --- a/rust/helm/hyperlane-agent/templates/validator-external-secret.yaml +++ b/rust/helm/hyperlane-agent/templates/validator-external-secret.yaml @@ -24,18 +24,18 @@ spec: {{ $index := 0 }} {{- range .Values.hyperlane.validator.configs }} validator-{{ $index }}.env: | -{{- if eq .validator.type "hexKey" }} - HYP_VALIDATOR_VALIDATOR_KEY={{ printf "'{{ .signer_key_%d | toString }}'" $index }} - HYP_BASE_CHAINS_{{ .originChainName | upper }}_SIGNER_KEY={{ printf "'{{ .signer_key_%d | toString }}'" $index }} -{{- end }} -{{- if or (eq .checkpointSyncer.type "s3") $.Values.hyperlane.aws }} + {{- if eq .validator.type "hexKey" }} + HYP_VALIDATOR_KEY={{ printf "'{{ .signer_key_%d | toString }}'" $index }} + HYP_CHAINS_{{ .originChainName | upper }}_SIGNER_KEY={{ printf "'{{ .signer_key_%d | toString }}'" $index }} + {{- end }} + {{- if or (eq .checkpointSyncer.type "s3") $.Values.hyperlane.aws }} AWS_ACCESS_KEY_ID={{ printf "'{{ .aws_access_key_id_%d | toString }}'" $index }} AWS_SECRET_ACCESS_KEY={{ printf "'{{ .aws_secret_access_key_%d | toString }}'" $index }} -{{- end }} + {{- end }} {{ $index = add1 $index }} {{- end }} data: -{{ $index := 0 }} +{{ $index = 0 }} {{- range .Values.hyperlane.validator.configs }} {{- if eq .validator.type "hexKey" }} - secretKey: signer_key_{{ $index }} diff --git a/rust/helm/hyperlane-agent/values.yaml b/rust/helm/hyperlane-agent/values.yaml index f2e15d0f5f..299cf0e63a 100644 --- a/rust/helm/hyperlane-agent/values.yaml +++ b/rust/helm/hyperlane-agent/values.yaml @@ -47,19 +47,33 @@ hyperlane: aws: # true | false # -- Chain overrides, a sequence + # This should mirror @hyperlane-xyz/sdk AgentChainMetadata chains: - - name: 'alfajores' + - name: examplechain disabled: false + rpcConsensusType: fallback signer: - # aws: - addresses: - mailbox: - multisigIsm: - interchainGasPaymaster: - domain: - protocol: # "ethereum" - connection: - type: # "http" + type: # aws + index: + from: + chunk: + mode: + mailbox: + multisigIsm: + interchainGasPaymaster: + interchainSecurityModule: + protocol: ethereum + chainId: + domainId: + customRpcUrls: + - example: + url: https://example.com + priority: 1 + blocks: + confirmations: + reorgPeriod: + estimatedBlockTime: + isTestnet: false # Hyperlane Agent Roles # Individually Switchable via .enabled @@ -81,7 +95,6 @@ hyperlane: # -- How long to wait between checking for updates configs: [] # - interval: - # reorgPeriod: # checkpointSyncers: # originChainName: # type: # "hexKey" @@ -103,6 +116,7 @@ hyperlane: cpu: 500m memory: 256Mi config: + relayChains: '' multisigCheckpointSyncer: checkpointSyncers: # -- Specify whether a default signer key is used for all chains in Values.hyperlane.relayerChains list. @@ -130,6 +144,7 @@ hyperlane: cpu: 250m memory: 256Mi config: + chainsToScrape: '' kathy: enabled: false diff --git a/rust/hyperlane-base/src/settings/deprecated_parser.rs b/rust/hyperlane-base/src/settings/deprecated_parser.rs deleted file mode 100644 index 4282f72d74..0000000000 --- a/rust/hyperlane-base/src/settings/deprecated_parser.rs +++ /dev/null @@ -1,435 +0,0 @@ -//! This module is responsible for parsing the agent's settings using the old config format. - -// TODO: Remove this module once we have finished migrating to the new format. - -use std::{ - collections::{HashMap, HashSet}, - path::PathBuf, -}; - -use ethers_prometheus::middleware::PrometheusMiddlewareConf; -use eyre::{eyre, Context}; -use hyperlane_core::{cfg_unwrap_all, config::*, utils::hex_or_base58_to_h256, HyperlaneDomain}; -use serde::Deserialize; - -use super::envs::*; -use crate::settings::{ - chains::IndexSettings, trace::TracingConfig, ChainConf, ChainConnectionConf, - CheckpointSyncerConf, CoreContractAddresses, Settings, SignerConf, -}; - -/// Raw base settings. -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DeprecatedRawSettings { - chains: Option>, - defaultsigner: Option, - metrics: Option, - tracing: Option, -} - -impl FromRawConf>> for Settings { - fn from_config_filtered( - raw: DeprecatedRawSettings, - cwp: &ConfigPath, - filter: Option<&HashSet<&str>>, - ) -> Result { - let mut err = ConfigParsingError::default(); - let chains: HashMap = if let Some(mut chains) = raw.chains { - let default_signer: Option = raw.defaultsigner.and_then(|r| { - r.parse_config(&cwp.join("defaultsigner")) - .take_config_err(&mut err) - }); - if let Some(filter) = filter { - chains.retain(|k, _| filter.contains(&k.as_str())); - } - let chains_path = cwp + "chains"; - chains - .into_iter() - .map(|(k, v)| { - let cwp = &chains_path + &k; - let k = k.to_ascii_lowercase(); - let mut parsed: ChainConf = v.parse_config(&cwp)?; - if let Some(default_signer) = &default_signer { - parsed.signer.get_or_insert_with(|| default_signer.clone()); - } - Ok((k, parsed)) - }) - .filter_map(|res| match res { - Ok((k, v)) => Some((k, v)), - Err(e) => { - err.merge(e); - None - } - }) - .collect() - } else { - Default::default() - }; - let tracing = raw.tracing.unwrap_or_default(); - let metrics = raw - .metrics - .and_then(|port| port.try_into().take_err(&mut err, || cwp + "metrics")) - .unwrap_or(9090); - - err.into_result(Self { - chains, - metrics_port: metrics, - tracing, - }) - } -} - -#[derive(Deserialize, Debug)] -#[serde(tag = "protocol", content = "connection", rename_all = "camelCase")] -enum DeprecatedRawChainConnectionConf { - Ethereum(h_eth::RawConnectionConf), - Fuel(h_fuel::DeprecatedRawConnectionConf), - Sealevel(h_sealevel::DeprecatedRawConnectionConf), - #[serde(other)] - Unknown, -} - -impl FromRawConf for ChainConnectionConf { - fn from_config_filtered( - raw: DeprecatedRawChainConnectionConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - use DeprecatedRawChainConnectionConf::*; - match raw { - Ethereum(r) => Ok(Self::Ethereum(r.parse_config(&cwp.join("connection"))?)), - Fuel(r) => Ok(Self::Fuel(r.parse_config(&cwp.join("connection"))?)), - Sealevel(r) => Ok(Self::Sealevel(r.parse_config(&cwp.join("connection"))?)), - Unknown => { - Err(eyre!("Unknown chain protocol")).into_config_result(|| cwp.join("protocol")) - } - } - } -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct DeprecatedRawCoreContractAddresses { - mailbox: Option, - interchain_gas_paymaster: Option, - validator_announce: Option, -} - -impl FromRawConf for CoreContractAddresses { - fn from_config_filtered( - raw: DeprecatedRawCoreContractAddresses, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - macro_rules! parse_addr { - ($name:ident) => { - let $name = raw - .$name - .ok_or_else(|| { - eyre!( - "Missing {} core contract address", - stringify!($name).replace('_', " ") - ) - }) - .take_err(&mut err, || cwp + stringify!($name)) - .and_then(|v| { - hex_or_base58_to_h256(&v).take_err(&mut err, || cwp + stringify!($name)) - }); - }; - } - - parse_addr!(mailbox); - parse_addr!(interchain_gas_paymaster); - parse_addr!(validator_announce); - - cfg_unwrap_all!(cwp, err: [mailbox, interchain_gas_paymaster, validator_announce]); - - err.into_result(Self { - mailbox, - interchain_gas_paymaster, - validator_announce, - }) - } -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct DeprecatedRawIndexSettings { - from: Option, - chunk: Option, - mode: Option, -} - -impl FromRawConf for IndexSettings { - fn from_config_filtered( - raw: DeprecatedRawIndexSettings, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let from = raw - .from - .and_then(|v| v.try_into().take_err(&mut err, || cwp + "from")) - .unwrap_or_default(); - - let chunk_size = raw - .chunk - .and_then(|v| v.try_into().take_err(&mut err, || cwp + "chunk")) - .unwrap_or(1999); - - let mode = raw - .mode - .map(serde_json::Value::from) - .and_then(|m| { - serde_json::from_value(m) - .context("Invalid mode") - .take_err(&mut err, || cwp + "mode") - }) - .unwrap_or_default(); - - err.into_result(Self { - from, - chunk_size, - mode, - }) - } -} - -/// A raw chain setup is a domain ID, an address on that chain (where the -/// mailbox is deployed) and details for connecting to the chain API. -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DeprecatedRawChainConf { - name: Option, - domain: Option, - pub(super) signer: Option, - finality_blocks: Option, - addresses: Option, - #[serde(flatten, default)] - connection: Option, - // TODO: if people actually use the metrics conf we should also add a raw form. - #[serde(default)] - metrics_conf: Option, - #[serde(default)] - index: Option, -} - -impl FromRawConf for ChainConf { - fn from_config_filtered( - raw: DeprecatedRawChainConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let connection = raw - .connection - .ok_or_else(|| eyre!("Missing `connection` configuration")) - .take_err(&mut err, || cwp + "connection") - .and_then(|r| r.parse_config(cwp).take_config_err(&mut err)); - - let domain = connection.as_ref().and_then(|c: &ChainConnectionConf| { - let protocol = c.protocol(); - let domain_id = raw - .domain - .ok_or_else(|| eyre!("Missing `domain` configuration")) - .take_err(&mut err, || cwp + "domain") - .and_then(|r| { - r.try_into() - .context("Invalid domain id, expected integer") - .take_err(&mut err, || cwp + "domain") - }); - let name = raw - .name - .as_deref() - .ok_or_else(|| eyre!("Missing domain `name` configuration")) - .take_err(&mut err, || cwp + "name"); - HyperlaneDomain::from_config(domain_id?, name?, protocol) - .take_err(&mut err, || cwp.clone()) - }); - - let addresses = raw - .addresses - .ok_or_else(|| eyre!("Missing `addresses` configuration for core contracts")) - .take_err(&mut err, || cwp + "addresses") - .and_then(|v| { - v.parse_config(&cwp.join("addresses")) - .take_config_err(&mut err) - }); - - let signer = raw.signer.and_then(|v| -> Option { - v.parse_config(&cwp.join("signer")) - .take_config_err(&mut err) - }); - - let finality_blocks = raw - .finality_blocks - .and_then(|v| { - v.try_into() - .context("Invalid `finalityBlocks`, expected integer") - .take_err(&mut err, || cwp + "finality_blocks") - }) - .unwrap_or(0); - - let index = raw - .index - .and_then(|v| v.parse_config(&cwp.join("index")).take_config_err(&mut err)) - .unwrap_or_default(); - - let metrics_conf = raw.metrics_conf.unwrap_or_default(); - - cfg_unwrap_all!(cwp, err: [connection, domain, addresses]); - - err.into_result(Self { - connection, - domain, - addresses, - signer, - finality_blocks, - index, - metrics_conf, - }) - } -} - -/// Raw signer types -#[derive(Debug, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -pub struct DeprecatedRawSignerConf { - #[serde(rename = "type")] - signer_type: Option, - key: Option, - id: Option, - region: Option, -} - -/// Raw checkpoint syncer types -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum DeprecatedRawCheckpointSyncerConf { - /// A local checkpoint syncer - LocalStorage { - /// Path - path: Option, - }, - /// A checkpoint syncer on S3 - S3 { - /// Bucket name - bucket: Option, - /// S3 Region - region: Option, - /// Folder name inside bucket - defaults to the root of the bucket - folder: Option, - }, - /// Unknown checkpoint syncer type was specified - #[serde(other)] - Unknown, -} - -impl FromRawConf for SignerConf { - fn from_config_filtered( - raw: DeprecatedRawSignerConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let key_path = || cwp + "key"; - let region_path = || cwp + "region"; - - match raw.signer_type.as_deref() { - Some("hexKey") => Ok(Self::HexKey { - key: raw - .key - .ok_or_else(|| eyre!("Missing `key` for HexKey signer")) - .into_config_result(key_path)? - .parse() - .into_config_result(key_path)?, - }), - Some("aws") => Ok(Self::Aws { - id: raw - .id - .ok_or_else(|| eyre!("Missing `id` for Aws signer")) - .into_config_result(|| cwp + "id")?, - region: raw - .region - .ok_or_else(|| eyre!("Missing `region` for Aws signer")) - .into_config_result(region_path)? - .parse() - .into_config_result(region_path)?, - }), - Some(t) => Err(eyre!("Unknown signer type `{t}`")).into_config_result(|| cwp + "type"), - None if raw.key.is_some() => Ok(Self::HexKey { - key: raw.key.unwrap().parse().into_config_result(key_path)?, - }), - None if raw.id.is_some() | raw.region.is_some() => Ok(Self::Aws { - id: raw - .id - .ok_or_else(|| eyre!("Missing `id` for Aws signer")) - .into_config_result(|| cwp + "id")?, - region: raw - .region - .ok_or_else(|| eyre!("Missing `region` for Aws signer")) - .into_config_result(region_path)? - .parse() - .into_config_result(region_path)?, - }), - None => Ok(Self::Node), - } - } -} - -impl FromRawConf for CheckpointSyncerConf { - fn from_config_filtered( - raw: DeprecatedRawCheckpointSyncerConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - match raw { - DeprecatedRawCheckpointSyncerConf::LocalStorage { path } => { - let path: PathBuf = path - .ok_or_else(|| eyre!("Missing `path` for LocalStorage checkpoint syncer")) - .into_config_result(|| cwp + "path")? - .parse() - .into_config_result(|| cwp + "path")?; - if !path.exists() { - std::fs::create_dir_all(&path) - .with_context(|| { - format!( - "Failed to create local checkpoint syncer storage directory at {:?}", - path - ) - }) - .into_config_result(|| cwp + "path")?; - } else if !path.is_dir() { - Err(eyre!( - "LocalStorage checkpoint syncer path is not a directory" - )) - .into_config_result(|| cwp + "path")?; - } - Ok(Self::LocalStorage { path }) - } - DeprecatedRawCheckpointSyncerConf::S3 { - bucket, - folder, - region, - } => Ok(Self::S3 { - bucket: bucket - .ok_or_else(|| eyre!("Missing `bucket` for S3 checkpoint syncer")) - .into_config_result(|| cwp + "bucket")?, - folder, - region: region - .ok_or_else(|| eyre!("Missing `region` for S3 checkpoint syncer")) - .into_config_result(|| cwp + "region")? - .parse() - .into_config_result(|| cwp + "region")?, - }), - DeprecatedRawCheckpointSyncerConf::Unknown => { - Err(eyre!("Missing `type` for checkpoint syncer")) - .into_config_result(|| cwp + "type") - } - } - } -} diff --git a/rust/hyperlane-base/src/settings/loader/arguments.rs b/rust/hyperlane-base/src/settings/loader/arguments.rs index 54f1b49f06..5cf1aeacf0 100644 --- a/rust/hyperlane-base/src/settings/loader/arguments.rs +++ b/rust/hyperlane-base/src/settings/loader/arguments.rs @@ -1,9 +1,7 @@ use std::ffi::{OsStr, OsString}; use config::{ConfigError, Map, Source, Value, ValueKind}; -use convert_case::Case; - -use crate::settings::loader::split_and_recase_key; +use itertools::Itertools; /// A source for loading configuration from command line arguments. /// @@ -24,10 +22,6 @@ pub struct CommandLineArguments { /// Ignore empty env values (treat as unset). ignore_empty: bool, - /// What casing to use for the keys in the environment. By default it will not mutate the key - /// value. - casing: Option, - /// Alternate source for the environment. This can be used when you want to /// test your own code using this source, without the need to change the /// actual system environment variables. @@ -46,11 +40,6 @@ impl CommandLineArguments { self } - pub fn casing(mut self, casing: Case) -> Self { - self.casing = Some(casing); - self - } - pub fn source(mut self, source: I) -> Self where I: IntoIterator, @@ -87,7 +76,7 @@ impl Source for CommandLineArguments { continue; } - let key = split_and_recase_key(separator, self.casing, key); + let key = key.split(separator).join("."); m.insert(key, Value::new(Some(&uri), ValueKind::String(value))); } diff --git a/rust/hyperlane-base/src/settings/loader/case_adapter.rs b/rust/hyperlane-base/src/settings/loader/case_adapter.rs new file mode 100644 index 0000000000..0f8e1e081e --- /dev/null +++ b/rust/hyperlane-base/src/settings/loader/case_adapter.rs @@ -0,0 +1,66 @@ +use std::fmt::Debug; + +use config::{ConfigError, Map, Source, Value, ValueKind}; +use convert_case::{Case, Casing}; +use derive_new::new; +use itertools::Itertools; + +#[derive(Clone, Debug, new)] +pub struct CaseAdapter { + inner: S, + casing: Case, +} + +impl Source for CaseAdapter +where + S: Source + Clone + Send + Sync + 'static, +{ + fn clone_into_box(&self) -> Box { + Box::new(self.clone()) + } + + fn collect(&self) -> Result, ConfigError> { + self.inner.collect().map(|m| { + m.into_iter() + .map(|(k, v)| recase_pair(k, v, self.casing)) + .collect() + }) + } +} + +fn recase_pair(key: String, mut val: Value, case: Case) -> (String, Value) { + let key = split_and_recase_key(".", Some(case), key); + match &mut val.kind { + ValueKind::Table(table) => { + let tmp = table + .drain() + .map(|(k, v)| recase_pair(k, v, case)) + .collect_vec(); + table.extend(tmp.into_iter()); + } + ValueKind::Array(ary) => { + let tmp = ary + .drain(..) + .map(|v| recase_pair(String::new(), v, case).1) + .collect_vec(); + ary.extend(tmp.into_iter()) + } + _ => {} + } + (key, val) +} + +/// Load a settings object from the config locations and re-join the components with the standard +/// `config` crate separator `.`. +fn split_and_recase_key(sep: &str, case: Option, key: String) -> String { + if let Some(case) = case { + // if case is given, replace case of each key component and separate them with `.` + key.split(sep).map(|s| s.to_case(case)).join(".") + } else if !sep.is_empty() && sep != "." { + // Just standardize the separator to `.` + key.replace(sep, ".") + } else { + // no changes needed if there was no separator defined and we are preserving case. + key + } +} diff --git a/rust/hyperlane-base/src/settings/loader/deprecated_arguments.rs b/rust/hyperlane-base/src/settings/loader/deprecated_arguments.rs deleted file mode 100644 index cc77e105c1..0000000000 --- a/rust/hyperlane-base/src/settings/loader/deprecated_arguments.rs +++ /dev/null @@ -1,343 +0,0 @@ -// TODO: Remove this file after deprecated config parsing has been removed. - -use std::ffi::{OsStr, OsString}; - -use config::{ConfigError, Map, Source, Value, ValueKind}; -use convert_case::Case; - -use crate::settings::loader::split_and_recase_key; - -/// A source for loading configuration from command line arguments. -/// Command line argument keys are case-insensitive, and the following forms are -/// supported: -/// -/// * `--key=value` -/// * `--key="value"` -/// * `--key='value'` -/// * `--key value` -/// * `--key` (value is an empty string) -#[must_use] -#[derive(Clone, Debug, Default)] -pub struct DeprecatedCommandLineArguments { - /// Optional character sequence that separates each key segment in an - /// environment key pattern. Consider a nested configuration such as - /// `redis.password`, a separator of `-` would allow an environment key - /// of `redis-password` to match. - separator: Option, - - /// Ignore empty env values (treat as unset). - ignore_empty: bool, - - /// Alternate source for the environment. This can be used when you want to - /// test your own code using this source, without the need to change the - /// actual system environment variables. - source: Option>, -} - -#[allow(unused)] -impl DeprecatedCommandLineArguments { - pub fn separator(mut self, s: &str) -> Self { - self.separator = Some(s.into()); - self - } - - pub fn ignore_empty(mut self, ignore: bool) -> Self { - self.ignore_empty = ignore; - self - } - - pub fn source(mut self, source: I) -> Self - where - I: IntoIterator, - S: AsRef, - { - self.source = Some(source.into_iter().map(|s| s.as_ref().to_owned()).collect()); - self - } -} - -impl Source for DeprecatedCommandLineArguments { - fn clone_into_box(&self) -> Box { - Box::new((*self).clone()) - } - - fn collect(&self) -> Result, ConfigError> { - let mut m = Map::new(); - let uri: String = "program argument".into(); - - let separator = self.separator.as_deref().unwrap_or("-"); - - let mut args = if let Some(source) = &self.source { - ArgumentParser::from_vec(source.clone()) - } else { - ArgumentParser::from_env() - }; - - while let Some((key, value)) = args - .next() - .transpose() - .map_err(|e| ConfigError::Foreign(Box::new(e)))? - { - if self.ignore_empty && value.is_empty() { - continue; - } - - let mut key = split_and_recase_key(separator, Some(Case::Flat), key); - if key.ends_with("interchaingaspaymaster") { - key = key.replace("interchaingaspaymaster", "interchainGasPaymaster"); - } else if key.ends_with("validatorannounce") { - key = key.replace("validatorannounce", "validatorAnnounce"); - } - - m.insert(key, Value::new(Some(&uri), ValueKind::String(value))); - } - - let remaining = args.finish(); - if remaining.is_empty() { - Ok(m) - } else { - Err(ConfigError::Message("Could not parse all arguments".into())) - } - } -} - -/// An ultra simple CLI arguments parser. -/// Adapted from pico-args 0.5.0. -#[derive(Clone, Debug)] -pub struct ArgumentParser(Vec); - -impl ArgumentParser { - /// Creates a parser from a vector of arguments. - /// - /// The executable path **must** be removed. - /// - /// This can be used for supporting `--` arguments to forward to another - /// program. - fn from_vec(args: Vec) -> Self { - ArgumentParser(args) - } - - /// Creates a parser from [`env::args_os`]. - /// - /// The executable path will be removed. - /// - /// [`env::args_os`]: https://doc.rust-lang.org/stable/std/env/fn.args_os.html - fn from_env() -> Self { - let mut args: Vec<_> = std::env::args_os().collect(); - args.remove(0); - ArgumentParser(args) - } - - /// Returns a list of remaining arguments. - /// - /// It's up to the caller what to do with them. - /// One can report an error about unused arguments, - /// other can use them for further processing. - fn finish(self) -> Vec { - self.0 - } -} - -impl Iterator for ArgumentParser { - type Item = Result<(String, String), Error>; - - fn next(&mut self) -> Option { - let (k, v, kind, idx) = match self.find_next_kv_pair() { - Ok(Some(tup)) => tup, - Ok(None) => return None, - Err(e) => return Some(Err(e)), - }; - - match kind { - PairKind::SingleArgument => { - self.0.remove(idx); - } - PairKind::TwoArguments => { - self.0.remove(idx + 1); - self.0.remove(idx); - } - } - - Some(Ok((k, v))) - } -} - -// internal workings -impl ArgumentParser { - #[inline(never)] - fn find_next_kv_pair(&mut self) -> Result, Error> { - let Some(idx) = self.index_of_next_key() else { - return Ok(None); - }; - // full term without leading '--' - let term = &os_to_str(&self.0[idx])?[2..]; - if term.is_empty() { - return Err(Error::EmptyKey); - } - - if let Some((key, value)) = term.split_once('=') { - // Parse a `--key=value` pair. - let key = key.to_owned(); - - // Check for quoted value. - let value = if starts_with(value, b'"') { - if !ends_with(value, b'"') { - // A closing quote must be the same as an opening one. - return Err(Error::UnmatchedQuote(key)); - } - &value[1..value.len() - 1] - } else if starts_with(value, b'\'') { - if !ends_with(value, b'\'') { - // A closing quote must be the same as an opening one. - return Err(Error::UnmatchedQuote(key)); - } - &value[1..value.len() - 1] - } else { - value - }; - - Ok(Some((key, value.to_owned(), PairKind::SingleArgument, idx))) - } else { - // Parse a `--key value` pair. - let key = term.to_owned(); - let value = self - .0 - .get(idx + 1) - .map(|v| os_to_str(v)) - .transpose()? - .unwrap_or(""); - - if value.is_empty() || value.starts_with('-') { - // the next value is another key - Ok(Some((key, "".to_owned(), PairKind::SingleArgument, idx))) - } else { - Ok(Some((key, value.to_owned(), PairKind::TwoArguments, idx))) - } - } - } - - fn index_of_next_key(&self) -> Option { - self.0.iter().position(|v| { - #[cfg(unix)] - { - use std::os::unix::ffi::OsStrExt; - v.len() >= 2 && &v.as_bytes()[0..2] == b"--" - } - #[cfg(not(unix))] - { - v.len() >= 2 && v.to_str().map(|v| v.starts_with("--")).unwrap_or(false) - } - }) - } -} - -#[inline] -fn starts_with(text: &str, c: u8) -> bool { - if text.is_empty() { - false - } else { - text.as_bytes()[0] == c - } -} - -#[inline] -fn ends_with(text: &str, c: u8) -> bool { - if text.is_empty() { - false - } else { - text.as_bytes()[text.len() - 1] == c - } -} - -#[inline] -fn os_to_str(text: &OsStr) -> Result<&str, Error> { - text.to_str().ok_or(Error::NonUtf8Argument) -} - -/// A list of possible errors. -#[derive(Clone, Debug, thiserror::Error)] -pub enum Error { - /// Arguments must be a valid UTF-8 strings. - #[error("argument is not a UTF-8 string")] - NonUtf8Argument, - - /// Found '--` or a key with nothing after the prefix - #[error("key name is empty (possibly after removing prefix)")] - EmptyKey, - - /// Could not find closing quote for a value. - #[error("unmatched quote in `{0}`")] - UnmatchedQuote(String), -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum PairKind { - SingleArgument, - TwoArguments, -} - -#[cfg(test)] -mod test { - use super::*; - - macro_rules! assert_arg { - ($config:expr, $key:literal, $value:literal) => { - let origin = "program argument".to_owned(); - assert_eq!( - $config.remove($key), - Some(Value::new( - Some(&origin), - ValueKind::String($value.to_owned()) - )) - ); - }; - } - - const ARGUMENTS: &[&str] = &[ - "--key-a", - "value-a", - "--keY-b=value-b", - "--key-c=\"value c\"", - "--KEY-d='valUE d'", - "--key-e=''", - "--key-F", - "--key-g=value-g", - "--key-h", - ]; - - #[test] - fn default_case() { - let mut config = DeprecatedCommandLineArguments::default() - .source(ARGUMENTS) - .collect() - .unwrap(); - - assert_arg!(config, "key.a", "value-a"); - assert_arg!(config, "key.b", "value-b"); - assert_arg!(config, "key.c", "value c"); - assert_arg!(config, "key.d", "valUE d"); - assert_arg!(config, "key.e", ""); - assert_arg!(config, "key.f", ""); - assert_arg!(config, "key.g", "value-g"); - assert_arg!(config, "key.h", ""); - - assert!(config.is_empty()); - } - - #[test] - fn ignore_empty() { - let mut config = DeprecatedCommandLineArguments::default() - .source(ARGUMENTS) - .ignore_empty(true) - .collect() - .unwrap(); - - assert_arg!(config, "key.a", "value-a"); - assert_arg!(config, "key.b", "value-b"); - assert_arg!(config, "key.c", "value c"); - assert_arg!(config, "key.d", "valUE d"); - assert_arg!(config, "key.g", "value-g"); - - assert!(config.is_empty()); - } -} diff --git a/rust/hyperlane-base/src/settings/loader/environment.rs b/rust/hyperlane-base/src/settings/loader/environment.rs index cd35744381..40d53e5a63 100644 --- a/rust/hyperlane-base/src/settings/loader/environment.rs +++ b/rust/hyperlane-base/src/settings/loader/environment.rs @@ -1,9 +1,7 @@ use std::env; use config::{ConfigError, Map, Source, Value, ValueKind}; -use convert_case::Case; - -use crate::settings::loader::split_and_recase_key; +use itertools::Itertools; #[must_use] #[derive(Clone, Debug, Default)] @@ -21,11 +19,6 @@ pub struct Environment { /// an environment key of `REDIS_PASSWORD` to match. Defaults to `_`. separator: Option, - /// What casing to use for the keys in the environment. By default it will not mutate the key - /// value. Case conversion will be performed after the prefix has been removed on each of the - /// seperated path components individually. - casing: Option, - /// Ignore empty env values (treat as unset). ignore_empty: bool, @@ -51,14 +44,9 @@ impl Environment { self } - pub fn casing(mut self, casing: Case) -> Self { - self.casing = Some(casing); - self - } - pub fn source<'a, I, S>(mut self, source: I) -> Self where - I: IntoIterator, + I: IntoIterator, S: AsRef + 'a, { self.source = Some( @@ -98,7 +86,7 @@ impl Source for Environment { return None; } - let key = split_and_recase_key(separator, self.casing, key); + let key = key.split(separator).join("."); Some((key, Value::new(Some(&uri), ValueKind::String(value)))) }; @@ -138,17 +126,16 @@ mod test { #[test] fn default_case() { let mut config = Environment::default() - .source(ENVS) + .source(ENVS.iter().cloned()) .prefix("PRE__") .separator("__") - .casing(Case::Camel) .collect() .unwrap(); - assert_env!(config, "key.a", "value-a"); + assert_env!(config, "KEY.A", "value-a"); assert_env!(config, "key.b", ""); - assert_env!(config, "key.c.partA", "value c a"); - assert_env!(config, "key.cPartB", "value c b"); + assert_env!(config, "KEY.C.PART_A", "value c a"); + assert_env!(config, "KEY.C_PART_B", "value c b"); assert!(config.is_empty()); } @@ -156,18 +143,17 @@ mod test { #[test] fn ignore_empty() { let mut config = Environment::default() - .source(ENVS) + .source(ENVS.iter().cloned()) .ignore_empty(true) - .source(ENVS) + .source(ENVS.iter().cloned()) .prefix("PRE__") .separator("__") - .casing(Case::Snake) .collect() .unwrap(); - assert_env!(config, "key.a", "value-a"); - assert_env!(config, "key.c.part_a", "value c a"); - assert_env!(config, "key.c_part_b", "value c b"); + assert_env!(config, "KEY.A", "value-a"); + assert_env!(config, "KEY.C.PART_A", "value c a"); + assert_env!(config, "KEY.C_PART_B", "value c b"); assert!(config.is_empty()); } diff --git a/rust/hyperlane-base/src/settings/loader/mod.rs b/rust/hyperlane-base/src/settings/loader/mod.rs index fe6ee9f34e..0286492733 100644 --- a/rust/hyperlane-base/src/settings/loader/mod.rs +++ b/rust/hyperlane-base/src/settings/loader/mod.rs @@ -1,49 +1,28 @@ //! Load a settings object from the config locations. -use std::{collections::HashMap, env, error::Error, fmt::Debug, path::PathBuf}; +use std::{env, error::Error, fmt::Debug, path::PathBuf}; -use config::{Config, Environment as DeprecatedEnvironment, File}; -use convert_case::{Case, Casing}; -use eyre::{bail, Context, Result}; +use config::{Config, File}; +use convert_case::Case; +use eyre::{eyre, Context, Result}; use hyperlane_core::config::*; -use itertools::Itertools; use serde::de::DeserializeOwned; -use crate::settings::loader::deprecated_arguments::DeprecatedCommandLineArguments; +use crate::settings::loader::{ + arguments::CommandLineArguments, case_adapter::CaseAdapter, environment::Environment, +}; mod arguments; -mod deprecated_arguments; +mod case_adapter; mod environment; /// Deserialize a settings object from the configs. -pub fn load_settings(name: &str) -> ConfigResult +pub fn load_settings() -> ConfigResult where T: DeserializeOwned + Debug, R: FromRawConf, { let root_path = ConfigPath::default(); - let raw = - load_settings_object::(name, &[]).into_config_result(|| root_path.clone())?; - raw.parse_config(&root_path) -} - -/// Load a settings object from the config locations. -/// Further documentation can be found in the `settings` module. -fn load_settings_object(agent_prefix: &str, ignore_prefixes: &[S]) -> Result -where - T: DeserializeOwned, - S: AsRef, -{ - // Derive additional prefix from agent name - let prefix = format!("HYP_{}", agent_prefix).to_ascii_uppercase(); - - let filtered_env: HashMap = env::vars() - .filter(|(k, _v)| { - !ignore_prefixes - .iter() - .any(|prefix| k.starts_with(prefix.as_ref())) - }) - .collect(); let mut base_config_sources = vec![]; let mut builder = Config::builder(); @@ -51,7 +30,8 @@ where // Always load the default config files (`rust/config/*.json`) for entry in PathBuf::from("./config") .read_dir() - .expect("Failed to open config directory") + .context("Failed to open config directory") + .into_config_result(|| root_path.clone())? .map(Result::unwrap) { if !entry.file_type().unwrap().is_file() { @@ -62,7 +42,7 @@ where let ext = fname.to_str().unwrap().split('.').last().unwrap_or(""); if ext == "json" { base_config_sources.push(format!("{:?}", entry.path())); - builder = builder.add_source(File::from(entry.path())); + builder = builder.add_source(CaseAdapter::new(File::from(entry.path()), Case::Flat)); } } @@ -75,31 +55,41 @@ where let p = PathBuf::from(path); if p.is_file() { if p.extension() == Some("json".as_ref()) { - builder = builder.add_source(File::from(p)); + let config_file = File::from(p); + let re_cased_config_file = CaseAdapter::new(config_file, Case::Flat); + builder = builder.add_source(re_cased_config_file); } else { - bail!("Provided config path via CONFIG_FILES is of an unsupported type ({p:?})") + return Err(eyre!( + "Provided config path via CONFIG_FILES is of an unsupported type ({p:?})" + )) + .into_config_result(|| root_path.clone()); } } else if !p.exists() { - bail!("Provided config path via CONFIG_FILES does not exist ({p:?})") + return Err(eyre!( + "Provided config path via CONFIG_FILES does not exist ({p:?})" + )) + .into_config_result(|| root_path.clone()); } else { - bail!("Provided config path via CONFIG_FILES is not a file ({p:?})") + return Err(eyre!( + "Provided config path via CONFIG_FILES is not a file ({p:?})" + )) + .into_config_result(|| root_path.clone()); } } let config_deserializer = builder // Use a base configuration env variable prefix - .add_source( - DeprecatedEnvironment::with_prefix("HYP_BASE") - .separator("_") - .source(Some(filtered_env.clone())), - ) - .add_source( - DeprecatedEnvironment::with_prefix(&prefix) - .separator("_") - .source(Some(filtered_env)), - ) - .add_source(DeprecatedCommandLineArguments::default().separator(".")) - .build()?; + .add_source(CaseAdapter::new( + Environment::default().prefix("HYP_").separator("_"), + Case::Flat, + )) + .add_source(CaseAdapter::new( + CommandLineArguments::default().separator("."), + Case::Flat, + )) + .build() + .context("Failed to load config sources") + .into_config_result(|| root_path.clone())?; let formatted_config = { let f = format!("{config_deserializer:#?}"); @@ -114,34 +104,26 @@ where } }; - Config::try_deserialize::(config_deserializer).or_else(|err| { - let mut err = if let Some(source_err) = err.source() { - let source = format!("Config error source: {source_err}"); - Err(err).context(source) - } else { - Err(err.into()) - }; - - for cfg_path in base_config_sources.iter().chain(config_file_paths.iter()) { - err = err.with_context(|| format!("Config loaded: {cfg_path}")); - } + let raw_config = Config::try_deserialize::(config_deserializer) + .or_else(|err| { + let mut err = if let Some(source_err) = err.source() { + let source = format!("Config error source: {source_err}"); + Err(err).context(source) + } else { + Err(err.into()) + }; - println!("Error during deserialization, showing the config for debugging: {formatted_config}"); - err.context("Config deserialization error, please check the config reference (https://docs.hyperlane.xyz/docs/operators/agent-configuration/configuration-reference)") - }) -} + for cfg_path in base_config_sources.iter().chain(config_file_paths.iter()) { + err = err.with_context(|| format!("Config loaded: {cfg_path}")); + } + eprintln!("Loaded config for debugging: {formatted_config}"); + err.context("Config deserialization error, please check the config reference (https://docs.hyperlane.xyz/docs/operators/agent-configuration/configuration-reference)") + }) + .into_config_result(|| root_path.clone())?; -/// Load a settings object from the config locations and re-join the components with the standard -/// `config` crate separator `.`. -fn split_and_recase_key(sep: &str, case: Option, key: String) -> String { - if let Some(case) = case { - // if case is given, replace case of each key component and separate them with `.` - key.split(sep).map(|s| s.to_case(case)).join(".") - } else if !sep.is_empty() && sep != "." { - // Just standardize the separator to `.` - key.replace(sep, ".") - } else { - // no changes needed if there was no separator defined and we are preserving case. - key + let res = raw_config.parse_config(&root_path); + if res.is_err() { + eprintln!("Loaded config for debugging: {formatted_config}"); } + res } diff --git a/rust/hyperlane-base/src/settings/mod.rs b/rust/hyperlane-base/src/settings/mod.rs index 70a617b362..ad69df0350 100644 --- a/rust/hyperlane-base/src/settings/mod.rs +++ b/rust/hyperlane-base/src/settings/mod.rs @@ -25,14 +25,7 @@ //! #### N.B.: Environment variable names correspond 1:1 with cfg file's JSON object hierarchy. //! //! In particular, note that any environment variables whose names are prefixed -//! with: -//! -//! * `HYP_BASE` -//! -//! * `HYP_[agentname]`, where `[agentmame]` is agent-specific, e.g. -//! `HYP_VALIDATOR` or `HYP_RELAYER`. -//! -//! will be read as an override to be applied against the hierarchical structure +//! with `HYP_` will be read as an override to be applied against the hierarchical structure //! of the configuration provided by the json config file at //! `./config//.json`. //! @@ -40,11 +33,10 @@ //! //! ```json //! { -//! "environment": "test", //! "signers": {}, //! "chains": { //! "test2": { -//! "domain": "13372", +//! "domainId": "13372", //! ... //! }, //! ... @@ -53,11 +45,9 @@ //! ``` //! //! and an environment variable is supplied which defines -//! `HYP_BASE_CHAINS_TEST2_DOMAIN=1`, then the `decl_settings` macro in -//! `rust/hyperlane-base/src/macros.rs` will directly override the 'domain' -//! field found in the json config to be `1`, since the fields in the -//! environment variable name describe the path traversal to arrive at this -//! field in the JSON config object. +//! `HYP_BASE_CHAINS_TEST2_DOMAINID=1`, then the config parser will directly override the value of +//! the field found in config to be `1`, since the fields in the environment variable name describe +//! the path traversal to arrive at this field in the JSON config object. //! //! ### Configuration value precedence //! @@ -69,10 +59,7 @@ //! overwriting previous ones as appropriate. //! 3. Configuration env vars with the prefix `HYP_BASE` intended //! to be shared by multiple agents in the same environment -//! E.g. `export HYP_BASE_INBOXES_KOVAN_DOMAIN=3000` -//! 4. Configuration env vars with the prefix `HYP_` -//! intended to be used by a specific agent. -//! E.g. `export HYP_RELAYER_ORIGINCHAIN="ethereum"` +//! E.g. `export HYP_CHAINS_ARBITRUM_DOMAINID=3000` //! 5. Arguments passed to the agent on the command line. //! E.g. `--originChainName ethereum` @@ -103,7 +90,6 @@ mod signers; mod trace; mod checkpoint_syncer; -pub mod deprecated_parser; pub mod parser; /// Declare that an agent can be constructed from settings. @@ -117,9 +103,7 @@ macro_rules! impl_loadable_from_settings { ($agent:ident, $settingsparser:ident -> $settingsobj:ident) => { impl hyperlane_base::LoadableFromSettings for $settingsobj { fn load() -> hyperlane_core::config::ConfigResult { - hyperlane_base::settings::loader::load_settings::<$settingsparser, Self>( - stringify!($agent), - ) + hyperlane_base::settings::loader::load_settings::<$settingsparser, Self>() } } }; diff --git a/rust/hyperlane-base/src/settings/parser/json_value_parser.rs b/rust/hyperlane-base/src/settings/parser/json_value_parser.rs index cefc8ccaee..69c2fe3484 100644 --- a/rust/hyperlane-base/src/settings/parser/json_value_parser.rs +++ b/rust/hyperlane-base/src/settings/parser/json_value_parser.rs @@ -4,6 +4,7 @@ use convert_case::{Case, Casing}; use derive_new::new; use eyre::{eyre, Context}; use hyperlane_core::{config::*, utils::hex_or_base58_to_h256, H256, U256}; +use itertools::Itertools; use serde::de::{DeserializeOwned, StdError}; use serde_json::Value; @@ -26,7 +27,7 @@ impl<'v> ValueParser<'v> { /// Get a value at the given key and verify that it is present. pub fn get_key(&self, key: &str) -> ConfigResult> { - self.get_opt_key(key)? + self.get_opt_key(&key.to_case(Case::Flat))? .ok_or_else(|| eyre!("Expected key `{key}` to be defined")) .into_config_result(|| &self.cwp + key.to_case(Case::Snake)) } @@ -35,7 +36,7 @@ impl<'v> ValueParser<'v> { pub fn get_opt_key(&self, key: &str) -> ConfigResult>> { let cwp = &self.cwp + key.to_case(Case::Snake); match self.val { - Value::Object(obj) => Ok(obj.get(key).map(|val| Self { + Value::Object(obj) => Ok(obj.get(&key.to_case(Case::Flat)).map(|val| Self { val, cwp: cwp.clone(), })), @@ -45,6 +46,7 @@ impl<'v> ValueParser<'v> { } /// Create an iterator over all (key, value) tuples. + /// Be warned that keys will be in flat case. pub fn into_obj_iter( self, ) -> ConfigResult)> + 'v> { @@ -67,11 +69,40 @@ impl<'v> ValueParser<'v> { /// Create an iterator over all array elements. pub fn into_array_iter(self) -> ConfigResult>> { let cwp = self.cwp.clone(); + match self.val { Value::Array(arr) => Ok(arr.iter().enumerate().map(move |(i, v)| Self { val: v, cwp: &cwp + i.to_string(), - })), + })) + .map(|itr| Box::new(itr) as Box>>), + Value::Object(obj) => obj + .iter() + // convert all keys to a usize index of their position in the array + .map(|(k, v)| k.parse().map(|k| (k, v))) + // handle any errors during index parsing + .collect::, _>>() + .context("Expected array or array-like object where all keys are indexes; some keys are not indexes") + // sort by index + .map(|arr| arr.into_iter().sorted_unstable_by_key(|(k, _)| *k)) + // check that all indexes are present + .and_then(|itr| { + itr.clone() + .enumerate() + .all(|(expected, (actual, _))| expected == actual) + .then_some(itr) + .ok_or(eyre!( + "Expected array or array-like object where all keys are indexes; some indexes are missing" + )) + }) + // convert to an iterator of value parsers over the values + .map(|itr| { + itr.map(move |(i, v)| Self { + val: v, + cwp: &cwp + i.to_string(), + }) + }) + .map(|itr| Box::new(itr) as Box>>), _ => Err(eyre!("Expected an array type")), } .into_config_result(|| self.cwp) diff --git a/rust/hyperlane-base/src/settings/parser/mod.rs b/rust/hyperlane-base/src/settings/parser/mod.rs index 3153701513..cf7f2cecdb 100644 --- a/rust/hyperlane-base/src/settings/parser/mod.rs +++ b/rust/hyperlane-base/src/settings/parser/mod.rs @@ -4,14 +4,12 @@ //! and validations it defines are not applied here, we should mirror them. //! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK. -#![allow(dead_code)] // TODO(2214): remove before PR merge - use std::{ - cmp::Reverse, collections::{HashMap, HashSet}, default::Default, }; +use convert_case::{Case, Casing}; use eyre::{eyre, Context}; use hyperlane_core::{ cfg_unwrap_all, config::*, HyperlaneDomain, HyperlaneDomainProtocol, IndexMode, @@ -23,8 +21,8 @@ use serde_json::Value; pub use self::json_value_parser::ValueParser; pub use super::envs::*; use crate::settings::{ - chains::IndexSettings, parser::json_value_parser::ParseChain, trace::TracingConfig, ChainConf, - ChainConnectionConf, CoreContractAddresses, Settings, SignerConf, + chains::IndexSettings, trace::TracingConfig, ChainConf, ChainConnectionConf, + CoreContractAddresses, Settings, SignerConf, }; mod json_value_parser; @@ -83,10 +81,16 @@ impl FromRawConf>> for Settings { .and_then(parse_signer) .end(); + let default_rpc_consensus_type = p + .chain(&mut err) + .get_opt_key("defaultRpcConsensusType") + .parse_string() + .unwrap_or("fallback"); + let chains: HashMap = raw_chains .into_iter() .filter_map(|(name, chain)| { - parse_chain(chain, &name) + parse_chain(chain, &name, default_rpc_consensus_type) .take_config_err(&mut err) .map(|v| (name, v)) }) @@ -107,7 +111,11 @@ impl FromRawConf>> for Settings { } /// The chain name and ChainMetadata -fn parse_chain(chain: ValueParser, name: &str) -> ConfigResult { +fn parse_chain( + chain: ValueParser, + name: &str, + default_rpc_consensus_type: &str, +) -> ConfigResult { let mut err = ConfigParsingError::default(); let domain = parse_domain(chain.clone(), name).take_config_err(&mut err); @@ -117,8 +125,6 @@ fn parse_chain(chain: ValueParser, name: &str) -> ConfigResult { .and_then(parse_signer) .end(); - // TODO(2214): is it correct to define finality blocks as `confirmations` and not `reorgPeriod`? - // TODO(2214): should we rename `finalityBlocks` in ChainConf? let finality_blocks = chain .chain(&mut err) .get_opt_key("blocks") @@ -126,33 +132,37 @@ fn parse_chain(chain: ValueParser, name: &str) -> ConfigResult { .parse_u32() .unwrap_or(1); - let rpcs: Vec = - if let Some(custom_rpc_urls) = chain.get_opt_key("customRpcUrls").unwrap_or_default() { - // use the custom defined urls, sorted by highest prio first - custom_rpc_urls.chain(&mut err).into_obj_iter().map(|itr| { - itr.map(|(_, url)| { - ( - url.chain(&mut err) - .get_opt_key("priority") - .parse_i32() - .unwrap_or(0), - url, - ) - }) - .sorted_unstable_by_key(|(p, _)| Reverse(*p)) - .map(|(_, url)| url) - .collect() + let rpcs_base = chain + .chain(&mut err) + .get_key("rpcUrls") + .into_array_iter() + .map(|urls| { + urls.filter_map(|v| { + v.chain(&mut err) + .get_key("http") + .parse_from_str("Invalid http url") + .end() }) - } else { - // if no custom rpc urls are set, use the default rpc urls - chain - .chain(&mut err) - .get_key("rpcUrls") - .into_array_iter() - .map(Iterator::collect) - } + .collect_vec() + }) .unwrap_or_default(); + let rpc_overrides = chain + .chain(&mut err) + .get_opt_key("customRpcUrls") + .parse_string() + .end() + .map(|urls| { + urls.split(',') + .filter_map(|url| { + url.parse() + .take_err(&mut err, || &chain.cwp + "customRpcUrls") + }) + .collect_vec() + }); + + let rpcs = rpc_overrides.unwrap_or(rpcs_base); + if rpcs.is_empty() { err.push( &chain.cwp + "rpc_urls", @@ -213,52 +223,35 @@ fn parse_chain(chain: ValueParser, name: &str) -> ConfigResult { let connection: Option = match domain.domain_protocol() { HyperlaneDomainProtocol::Ethereum => { if rpcs.len() <= 1 { - let into_connection = - |url| ChainConnectionConf::Ethereum(h_eth::ConnectionConf::Http { url }); - rpcs.into_iter().next().and_then(|rpc| { - rpc.chain(&mut err) - .get_key("http") - .parse_from_str("Invalid http url") - .end() - .map(into_connection) - }) + rpcs.into_iter() + .next() + .map(|url| ChainConnectionConf::Ethereum(h_eth::ConnectionConf::Http { url })) } else { - let urls = rpcs - .into_iter() - .filter_map(|rpc| { - rpc.chain(&mut err) - .get_key("http") - .parse_from_str("Invalid http url") - .end() - }) - .collect_vec(); - let rpc_consensus_type = chain .chain(&mut err) .get_opt_key("rpcConsensusType") .parse_string() - .unwrap_or("fallback"); + .unwrap_or(default_rpc_consensus_type); match rpc_consensus_type { - "fallback" => Some(h_eth::ConnectionConf::HttpFallback { urls }), - "quorum" => Some(h_eth::ConnectionConf::HttpQuorum { urls }), + "single" => Some(h_eth::ConnectionConf::Http { + url: rpcs.into_iter().next().unwrap(), + }), + "fallback" => Some(h_eth::ConnectionConf::HttpFallback { urls: rpcs }), + "quorum" => Some(h_eth::ConnectionConf::HttpQuorum { urls: rpcs }), ty => Err(eyre!("unknown rpc consensus type `{ty}`")) .take_err(&mut err, || &chain.cwp + "rpc_consensus_type"), } .map(ChainConnectionConf::Ethereum) } } - HyperlaneDomainProtocol::Fuel => ParseChain::from_option(rpcs.into_iter().next(), &mut err) - .get_key("http") - .parse_from_str("Invalid http url") - .end() + HyperlaneDomainProtocol::Fuel => rpcs + .into_iter() + .next() .map(|url| ChainConnectionConf::Fuel(h_fuel::ConnectionConf { url })), - HyperlaneDomainProtocol::Sealevel => { - ParseChain::from_option(rpcs.into_iter().next(), &mut err) - .get_key("http") - .parse_from_str("Invalod http url") - .end() - .map(|url| ChainConnectionConf::Sealevel(h_sealevel::ConnectionConf { url })) - } + HyperlaneDomainProtocol::Sealevel => rpcs + .into_iter() + .next() + .map(|url| ChainConnectionConf::Sealevel(h_sealevel::ConnectionConf { url })), }; cfg_unwrap_all!(&chain.cwp, err: [connection, mailbox, interchain_gas_paymaster, validator_announce]); @@ -387,3 +380,24 @@ impl FromRawConf for SignerConf { parse_signer(ValueParser::new(cwp.clone(), &raw.0)) } } + +/// Recursively re-cases a json value's keys to the given case. +pub fn recase_json_value(mut val: Value, case: Case) -> Value { + match &mut val { + Value::Array(ary) => { + for i in ary { + let val = recase_json_value(i.take(), case); + *i = val; + } + } + Value::Object(obj) => { + let keys = obj.keys().cloned().collect_vec(); + for key in keys { + let val = obj.remove(&key).unwrap(); + obj.insert(key.to_case(case), recase_json_value(val, case)); + } + } + _ => {} + } + val +} diff --git a/rust/hyperlane-base/tests/chain_config.rs b/rust/hyperlane-base/tests/chain_config.rs index 7024f641b3..7e59b23606 100644 --- a/rust/hyperlane-base/tests/chain_config.rs +++ b/rust/hyperlane-base/tests/chain_config.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeSet, fs::read_to_string, path::Path}; use config::{Config, FileFormat}; use eyre::Context; -use hyperlane_base::settings::{deprecated_parser::DeprecatedRawSettings, Settings}; +use hyperlane_base::settings::{parser::RawAgentConf, Settings}; use hyperlane_core::{config::*, KnownHyperlaneDomain}; use walkdir::WalkDir; @@ -71,11 +71,11 @@ fn hyperlane_settings() -> Vec { .zip(files.iter()) // Filter out config files that can't be parsed as json (e.g. env files) .filter_map(|(p, f)| { - let raw: DeprecatedRawSettings = Config::builder() + let raw: RawAgentConf = Config::builder() .add_source(config::File::from_str(f.as_str(), FileFormat::Json)) .build() .ok()? - .try_deserialize::() + .try_deserialize::() .unwrap_or_else(|e| { panic!("!cfg({}): {:?}: {}", p, e, f); }); diff --git a/rust/hyperlane-core/src/config/config_path.rs b/rust/hyperlane-core/src/config/config_path.rs index 1126e0e82b..1d0af8def7 100644 --- a/rust/hyperlane-core/src/config/config_path.rs +++ b/rust/hyperlane-core/src/config/config_path.rs @@ -76,10 +76,10 @@ impl ConfigPath { /// Get the environment variable formatted path. pub fn env_name(&self) -> String { - ["HYP", "BASE"] + ["HYP"] .into_iter() .chain(self.0.iter().map(|s| s.as_str())) - .map(|s| s.to_uppercase()) + .map(|s| s.to_case(Case::UpperFlat)) .join("_") } diff --git a/rust/sealevel/environments/local-e2e/warp-routes/testwarproute/program-ids.json b/rust/sealevel/environments/local-e2e/warp-routes/testwarproute/program-ids.json index c5e945eae3..ba62748efe 100644 --- a/rust/sealevel/environments/local-e2e/warp-routes/testwarproute/program-ids.json +++ b/rust/sealevel/environments/local-e2e/warp-routes/testwarproute/program-ids.json @@ -1,10 +1,10 @@ { - "sealeveltest2": { - "hex": "0x2317f9615d4ebc2419ad4b88580e2a80a03b2c7a60bc960de7d6934dbc37a87e", - "base58": "3MzUPjP5LEkiHH82nEAe28Xtz9ztuMqWc8UmuKxrpVQH" - }, "sealeveltest1": { "hex": "0xa77b4e2ed231894cc8cb8eee21adcc705d8489bccc6b2fcf40a358de23e60b7b", "base58": "CGn8yNtSD3aTTqJfYhUb6s1aVTN75NzwtsFKo1e83aga" + }, + "sealeveltest2": { + "hex": "0x2317f9615d4ebc2419ad4b88580e2a80a03b2c7a60bc960de7d6934dbc37a87e", + "base58": "3MzUPjP5LEkiHH82nEAe28Xtz9ztuMqWc8UmuKxrpVQH" } } \ No newline at end of file diff --git a/rust/utils/run-locally/src/main.rs b/rust/utils/run-locally/src/main.rs index 72574ca9f1..1c3c9a4e4c 100644 --- a/rust/utils/run-locally/src/main.rs +++ b/rust/utils/run-locally/src/main.rs @@ -12,26 +12,27 @@ //! - `E2E_KATHY_MESSAGES`: Number of kathy messages to dispatch. Defaults to 16 if CI mode is enabled. //! else false. -use std::path::Path; use std::{ fs, + path::Path, process::{Child, ExitCode}, sync::atomic::{AtomicBool, Ordering}, thread::sleep, time::{Duration, Instant}, }; -use tempfile::tempdir; - use logging::log; pub use metrics::fetch_metric; use program::Program; +use tempfile::tempdir; -use crate::config::Config; -use crate::ethereum::start_anvil; -use crate::invariants::{termination_invariants_met, SOL_MESSAGES_EXPECTED}; -use crate::solana::*; -use crate::utils::{concat_path, make_static, stop_child, AgentHandles, ArbitraryData, TaskHandle}; +use crate::{ + config::Config, + ethereum::start_anvil, + invariants::{termination_invariants_met, SOL_MESSAGES_EXPECTED}, + solana::*, + utils::{concat_path, make_static, stop_child, AgentHandles, ArbitraryData, TaskHandle}, +}; mod config; mod ethereum; @@ -145,8 +146,8 @@ fn main() -> ExitCode { let common_agent_env = Program::default() .env("RUST_BACKTRACE", "full") - .hyp_env("TRACING_FMT", "compact") - .hyp_env("TRACING_LEVEL", "debug") + .hyp_env("LOG_FORMAT", "compact") + .hyp_env("LOG_LEVEL", "debug") .hyp_env("CHAINS_TEST1_INDEX_CHUNK", "1") .hyp_env("CHAINS_TEST2_INDEX_CHUNK", "1") .hyp_env("CHAINS_TEST3_INDEX_CHUNK", "1"); @@ -154,16 +155,16 @@ fn main() -> ExitCode { let relayer_env = common_agent_env .clone() .bin(concat_path(AGENT_BIN_PATH, "relayer")) - .hyp_env("CHAINS_TEST1_CONNECTION_TYPE", "httpFallback") + .hyp_env("CHAINS_TEST1_RPCCONSENSUSTYPE", "fallback") .hyp_env( "CHAINS_TEST2_CONNECTION_URLS", "http://127.0.0.1:8545,http://127.0.0.1:8545,http://127.0.0.1:8545", ) // by setting this as a quorum provider we will cause nonce errors when delivering to test2 // because the message will be sent to the node 3 times. - .hyp_env("CHAINS_TEST2_CONNECTION_TYPE", "httpQuorum") - .hyp_env("CHAINS_TEST3_CONNECTION_URL", "http://127.0.0.1:8545") - .hyp_env("METRICS", "9092") + .hyp_env("CHAINS_TEST2_RPCCONSENSUSTYPE", "quorum") + .hyp_env("CHAINS_TEST3_RPCCONSENSUSTYPE", "http://127.0.0.1:8545") + .hyp_env("METRICSPORT", "9092") .hyp_env("DB", relayer_db.to_str().unwrap()) .hyp_env("CHAINS_TEST1_SIGNER_KEY", RELAYER_KEYS[0]) .hyp_env("CHAINS_TEST2_SIGNER_KEY", RELAYER_KEYS[1]) @@ -188,7 +189,7 @@ fn main() -> ExitCode { }]"#, ) .arg( - "chains.test1.connection.urls", + "chains.test1.customRpcUrls", "http://127.0.0.1:8545,http://127.0.0.1:8545,http://127.0.0.1:8545", ) // default is used for TEST3 @@ -202,17 +203,19 @@ fn main() -> ExitCode { .clone() .bin(concat_path(AGENT_BIN_PATH, "validator")) .hyp_env( - "CHAINS_TEST1_CONNECTION_URLS", + "CHAINS_TEST1_CUSTOMRPCURLS", "http://127.0.0.1:8545,http://127.0.0.1:8545,http://127.0.0.1:8545", ) - .hyp_env("CHAINS_TEST1_CONNECTION_TYPE", "httpQuorum") + .hyp_env("CHAINS_TEST1_RPCCONSENSUSTYPE", "quorum") .hyp_env( - "CHAINS_TEST2_CONNECTION_URLS", + "CHAINS_TEST2_CUSTOMRPCURLS", "http://127.0.0.1:8545,http://127.0.0.1:8545,http://127.0.0.1:8545", ) - .hyp_env("CHAINS_TEST2_CONNECTION_TYPE", "httpFallback") - .hyp_env("CHAINS_TEST3_CONNECTION_URL", "http://127.0.0.1:8545") - .hyp_env("REORGPERIOD", "0") + .hyp_env("CHAINS_TEST2_RPCCONSENSUSTYPE", "fallback") + .hyp_env("CHAINS_TEST3_CUSTOMRPCURLS", "http://127.0.0.1:8545") + .hyp_env("CHAINS_TEST1_BLOCKS_REORGPERIOD", "0") + .hyp_env("CHAINS_TEST2_BLOCKS_REORGPERIOD", "0") + .hyp_env("CHAINS_TEST3_BLOCKS_REORGPERIOD", "0") .hyp_env("INTERVAL", "5") .hyp_env("CHECKPOINTSYNCER_TYPE", "localStorage"); @@ -220,7 +223,7 @@ fn main() -> ExitCode { .map(|i| { base_validator_env .clone() - .hyp_env("METRICS", (9094 + i).to_string()) + .hyp_env("METRICSPORT", (9094 + i).to_string()) .hyp_env("DB", validator_dbs[i].to_str().unwrap()) .hyp_env("ORIGINCHAINNAME", VALIDATOR_ORIGIN_CHAINS[i]) .hyp_env("VALIDATOR_KEY", VALIDATOR_KEYS[i]) @@ -233,14 +236,14 @@ fn main() -> ExitCode { let scraper_env = common_agent_env .bin(concat_path(AGENT_BIN_PATH, "scraper")) - .hyp_env("CHAINS_TEST1_CONNECTION_TYPE", "httpQuorum") - .hyp_env("CHAINS_TEST1_CONNECTION_URL", "http://127.0.0.1:8545") - .hyp_env("CHAINS_TEST2_CONNECTION_TYPE", "httpQuorum") - .hyp_env("CHAINS_TEST2_CONNECTION_URL", "http://127.0.0.1:8545") - .hyp_env("CHAINS_TEST3_CONNECTION_TYPE", "httpQuorum") - .hyp_env("CHAINS_TEST3_CONNECTION_URL", "http://127.0.0.1:8545") + .hyp_env("CHAINS_TEST1_RPCCONSENSUSTYPE", "quorum") + .hyp_env("CHAINS_TEST1_CUSTOMRPCURLS", "http://127.0.0.1:8545") + .hyp_env("CHAINS_TEST2_RPCCONSENSUSTYPE", "quorum") + .hyp_env("CHAINS_TEST2_CUSTOMRPCURLS", "http://127.0.0.1:8545") + .hyp_env("CHAINS_TEST3_RPCCONSENSUSTYPE", "quorum") + .hyp_env("CHAINS_TEST3_CUSTOMRPCURLS", "http://127.0.0.1:8545") .hyp_env("CHAINSTOSCRAPE", "test1,test2,test3") - .hyp_env("METRICS", "9093") + .hyp_env("METRICSPORT", "9093") .hyp_env( "DB", "postgresql://postgres:47221c18c610@localhost:5432/postgres", diff --git a/rust/utils/run-locally/src/program.rs b/rust/utils/run-locally/src/program.rs index 5a27d1c484..5c2768ae1c 100644 --- a/rust/utils/run-locally/src/program.rs +++ b/rust/utils/run-locally/src/program.rs @@ -1,24 +1,31 @@ -use std::collections::BTreeMap; -use std::ffi::OsStr; -use std::fmt::{Debug, Display, Formatter}; -use std::io::{BufRead, BufReader, Read}; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::Sender; -use std::sync::{mpsc, Arc}; -use std::thread::{sleep, spawn}; -use std::time::Duration; +use std::{ + collections::BTreeMap, + ffi::OsStr, + fmt::{Debug, Display, Formatter}, + io::{BufRead, BufReader, Read}, + path::{Path, PathBuf}, + process::{Command, Stdio}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, + mpsc::Sender, + Arc, + }, + thread::{sleep, spawn}, + time::Duration, +}; use eyre::Context; use macro_rules_attribute::apply; -use crate::logging::log; -use crate::utils::{ - as_task, stop_child, AgentHandles, ArbitraryData, LogFilter, MappingTaskHandle, - SimpleTaskHandle, TaskHandle, +use crate::{ + logging::log, + utils::{ + as_task, stop_child, AgentHandles, ArbitraryData, LogFilter, MappingTaskHandle, + SimpleTaskHandle, TaskHandle, + }, + RUN_LOG_WATCHERS, SHUTDOWN, }; -use crate::{RUN_LOG_WATCHERS, SHUTDOWN}; #[derive(Default, Clone)] #[must_use] @@ -134,7 +141,7 @@ impl Program { /// add an env that will be prefixed with the default hyperlane env prefix pub fn hyp_env(self, key: impl AsRef, value: impl Into) -> Self { - const PREFIX: &str = "HYP_BASE_"; + const PREFIX: &str = "HYP_"; let key = key.as_ref(); debug_assert!( !key.starts_with(PREFIX), diff --git a/typescript/infra/config/environments/mainnet2/agent.ts b/typescript/infra/config/environments/mainnet2/agent.ts index c6bdded502..c7941a48f4 100644 --- a/typescript/infra/config/environments/mainnet2/agent.ts +++ b/typescript/infra/config/environments/mainnet2/agent.ts @@ -1,12 +1,12 @@ import { - AgentConnectionType, + GasPaymentEnforcementPolicyType, + RpcConsensusType, chainMetadata, hyperlaneEnvironments, } from '@hyperlane-xyz/sdk'; import { objMap } from '@hyperlane-xyz/utils'; import { - GasPaymentEnforcementPolicyType, RootAgentConfig, allAgentChainNames, routerMatchingList, @@ -80,7 +80,7 @@ const hyperlane: RootAgentConfig = { context: Contexts.Hyperlane, rolesWithKeys: ALL_KEY_ROLES, relayer: { - connectionType: AgentConnectionType.HttpFallback, + rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, tag: '3b0685f-20230815-110725', @@ -117,11 +117,11 @@ const hyperlane: RootAgentConfig = { tag: '3b0685f-20230815-110725', }, }, - connectionType: AgentConnectionType.HttpQuorum, + rpcConsensusType: RpcConsensusType.Quorum, chains: validatorChainConfig(Contexts.Hyperlane), }, scraper: { - connectionType: AgentConnectionType.HttpFallback, + rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, tag: 'aaddba7-20230620-154941', @@ -134,7 +134,7 @@ const releaseCandidate: RootAgentConfig = { context: Contexts.ReleaseCandidate, rolesWithKeys: [Role.Relayer, Role.Kathy, Role.Validator], relayer: { - connectionType: AgentConnectionType.HttpFallback, + rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, tag: '3b0685f-20230815-110725', @@ -144,14 +144,14 @@ const releaseCandidate: RootAgentConfig = { transactionGasLimit: 750000, // Skipping arbitrum because the gas price estimates are inclusive of L1 // fees which leads to wildly off predictions. - skipTransactionGasLimitFor: [chainMetadata.arbitrum.chainId], + skipTransactionGasLimitFor: [chainMetadata.arbitrum.name], }, validators: { docker: { repo, tag: 'ed7569d-20230725-171222', }, - connectionType: AgentConnectionType.HttpQuorum, + rpcConsensusType: RpcConsensusType.Quorum, chains: validatorChainConfig(Contexts.ReleaseCandidate), }, }; diff --git a/typescript/infra/config/environments/mainnet2/funding.ts b/typescript/infra/config/environments/mainnet2/funding.ts index 969b75fbac..7bfa7a2d9b 100644 --- a/typescript/infra/config/environments/mainnet2/funding.ts +++ b/typescript/infra/config/environments/mainnet2/funding.ts @@ -1,4 +1,4 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; +import { RpcConsensusType } from '@hyperlane-xyz/sdk'; import { KeyFunderConfig } from '../../../src/config/funding'; import { Role } from '../../../src/roles'; @@ -23,5 +23,5 @@ export const keyFunderConfig: KeyFunderConfig = { [Contexts.Hyperlane]: [Role.Relayer, Role.Kathy], [Contexts.ReleaseCandidate]: [Role.Relayer, Role.Kathy], }, - connectionType: AgentConnectionType.Http, + connectionType: RpcConsensusType.Single, }; diff --git a/typescript/infra/config/environments/mainnet2/helloworld.ts b/typescript/infra/config/environments/mainnet2/helloworld.ts index dbc6cee4aa..c9a803ef23 100644 --- a/typescript/infra/config/environments/mainnet2/helloworld.ts +++ b/typescript/infra/config/environments/mainnet2/helloworld.ts @@ -1,4 +1,4 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; +import { RpcConsensusType } from '@hyperlane-xyz/sdk'; import { HelloWorldConfig } from '../../../src/config'; import { HelloWorldKathyRunMode } from '../../../src/config/helloworld'; @@ -24,7 +24,7 @@ export const hyperlane: HelloWorldConfig = { }, messageSendTimeout: 1000 * 60 * 8, // 8 min messageReceiptTimeout: 1000 * 60 * 20, // 20 min - connectionType: AgentConnectionType.HttpFallback, + connectionType: RpcConsensusType.Fallback, cyclesBetweenEthereumMessages: 3, // Skip 3 cycles of Ethereum, i.e. send/receive Ethereum messages every 32 hours. }, }; @@ -44,7 +44,7 @@ export const releaseCandidate: HelloWorldConfig = { }, messageSendTimeout: 1000 * 60 * 8, // 8 min messageReceiptTimeout: 1000 * 60 * 20, // 20 min - connectionType: AgentConnectionType.Http, + connectionType: RpcConsensusType.Single, }, }; diff --git a/typescript/infra/config/environments/mainnet2/index.ts b/typescript/infra/config/environments/mainnet2/index.ts index 17b2839309..692d0f31dd 100644 --- a/typescript/infra/config/environments/mainnet2/index.ts +++ b/typescript/infra/config/environments/mainnet2/index.ts @@ -1,4 +1,4 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; +import { RpcConsensusType } from '@hyperlane-xyz/sdk'; import { getKeysForRole, @@ -26,7 +26,7 @@ export const environment: EnvironmentConfig = { getMultiProvider: ( context: Contexts = Contexts.Hyperlane, role: Role = Role.Deployer, - connectionType?: AgentConnectionType, + connectionType?: RpcConsensusType, ) => getMultiProviderForRole( mainnetConfigs, diff --git a/typescript/infra/config/environments/mainnet2/liquidityLayer.ts b/typescript/infra/config/environments/mainnet2/liquidityLayer.ts index a8779cf903..e81d7564a8 100644 --- a/typescript/infra/config/environments/mainnet2/liquidityLayer.ts +++ b/typescript/infra/config/environments/mainnet2/liquidityLayer.ts @@ -1,9 +1,9 @@ import { - AgentConnectionType, BridgeAdapterConfig, BridgeAdapterType, ChainMap, Chains, + RpcConsensusType, chainMetadata, } from '@hyperlane-xyz/sdk'; @@ -45,5 +45,5 @@ export const relayerConfig: LiquidityLayerRelayerConfig = { namespace: environment, prometheusPushGateway: 'http://prometheus-pushgateway.monitoring.svc.cluster.local:9091', - connectionType: AgentConnectionType.Http, + connectionType: RpcConsensusType.Single, }; diff --git a/typescript/infra/config/environments/test/agent.ts b/typescript/infra/config/environments/test/agent.ts index c4516d88e5..ff822528e1 100644 --- a/typescript/infra/config/environments/test/agent.ts +++ b/typescript/infra/config/environments/test/agent.ts @@ -1,9 +1,9 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; - import { GasPaymentEnforcementPolicyType, - RootAgentConfig, -} from '../../../src/config'; + RpcConsensusType, +} from '@hyperlane-xyz/sdk'; + +import { RootAgentConfig } from '../../../src/config'; import { ALL_KEY_ROLES } from '../../../src/roles'; import { Contexts } from '../../contexts'; @@ -15,7 +15,7 @@ const roleBase = { repo: 'gcr.io/abacus-labs-dev/hyperlane-agent', tag: '8852db3d88e87549269487da6da4ea5d67fdbfed', }, - connectionType: AgentConnectionType.Http, + rpcConsensusType: RpcConsensusType.Single, } as const; const hyperlane: RootAgentConfig = { diff --git a/typescript/infra/config/environments/testnet3/agent.ts b/typescript/infra/config/environments/testnet3/agent.ts index 0b31b7081a..d5c79df715 100644 --- a/typescript/infra/config/environments/testnet3/agent.ts +++ b/typescript/infra/config/environments/testnet3/agent.ts @@ -1,5 +1,6 @@ import { - AgentConnectionType, + GasPaymentEnforcementPolicyType, + RpcConsensusType, chainMetadata, getDomainId, hyperlaneEnvironments, @@ -7,7 +8,6 @@ import { import { objMap } from '@hyperlane-xyz/utils'; import { - GasPaymentEnforcementPolicyType, RootAgentConfig, allAgentChainNames, routerMatchingList, @@ -71,7 +71,7 @@ const hyperlane: RootAgentConfig = { context: Contexts.Hyperlane, rolesWithKeys: ALL_KEY_ROLES, relayer: { - connectionType: AgentConnectionType.HttpFallback, + rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, tag: 'ed7569d-20230725-171222', @@ -88,7 +88,7 @@ const hyperlane: RootAgentConfig = { gasPaymentEnforcement, }, validators: { - connectionType: AgentConnectionType.HttpFallback, + rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, tag: 'ed7569d-20230725-171222', @@ -104,7 +104,7 @@ const hyperlane: RootAgentConfig = { chains: validatorChainConfig(Contexts.Hyperlane), }, scraper: { - connectionType: AgentConnectionType.HttpFallback, + rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, tag: 'aaddba7-20230620-154941', @@ -117,7 +117,7 @@ const releaseCandidate: RootAgentConfig = { context: Contexts.ReleaseCandidate, rolesWithKeys: [Role.Relayer, Role.Kathy, Role.Validator], relayer: { - connectionType: AgentConnectionType.HttpFallback, + rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, tag: 'c7c44b2-20230811-133851', @@ -175,10 +175,10 @@ const releaseCandidate: RootAgentConfig = { transactionGasLimit: 750000, // Skipping arbitrum because the gas price estimates are inclusive of L1 // fees which leads to wildly off predictions. - skipTransactionGasLimitFor: [chainMetadata.arbitrumgoerli.chainId], + skipTransactionGasLimitFor: [chainMetadata.arbitrumgoerli.name], }, validators: { - connectionType: AgentConnectionType.HttpFallback, + rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, tag: 'ed7569d-20230725-171222', diff --git a/typescript/infra/config/environments/testnet3/funding.ts b/typescript/infra/config/environments/testnet3/funding.ts index 4c5491cadd..4f1e4280ba 100644 --- a/typescript/infra/config/environments/testnet3/funding.ts +++ b/typescript/infra/config/environments/testnet3/funding.ts @@ -1,4 +1,4 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; +import { RpcConsensusType } from '@hyperlane-xyz/sdk'; import { KeyFunderConfig } from '../../../src/config/funding'; import { Role } from '../../../src/roles'; @@ -23,5 +23,5 @@ export const keyFunderConfig: KeyFunderConfig = { [Contexts.Hyperlane]: [Role.Relayer, Role.Kathy], [Contexts.ReleaseCandidate]: [Role.Relayer, Role.Kathy], }, - connectionType: AgentConnectionType.HttpQuorum, + connectionType: RpcConsensusType.Quorum, }; diff --git a/typescript/infra/config/environments/testnet3/helloworld.ts b/typescript/infra/config/environments/testnet3/helloworld.ts index f6e50e030c..99fb4738f2 100644 --- a/typescript/infra/config/environments/testnet3/helloworld.ts +++ b/typescript/infra/config/environments/testnet3/helloworld.ts @@ -1,4 +1,4 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; +import { RpcConsensusType } from '@hyperlane-xyz/sdk'; import { HelloWorldConfig } from '../../../src/config'; import { HelloWorldKathyRunMode } from '../../../src/config/helloworld'; @@ -24,7 +24,7 @@ export const hyperlaneHelloworld: HelloWorldConfig = { }, messageSendTimeout: 1000 * 60 * 8, // 8 min messageReceiptTimeout: 1000 * 60 * 20, // 20 min - connectionType: AgentConnectionType.HttpFallback, + connectionType: RpcConsensusType.Fallback, }, }; @@ -43,7 +43,7 @@ export const releaseCandidateHelloworld: HelloWorldConfig = { }, messageSendTimeout: 1000 * 60 * 8, // 8 min messageReceiptTimeout: 1000 * 60 * 20, // 20 min - connectionType: AgentConnectionType.Http, + connectionType: RpcConsensusType.Single, }, }; diff --git a/typescript/infra/config/environments/testnet3/index.ts b/typescript/infra/config/environments/testnet3/index.ts index 9bb2134b6c..d82aa9995f 100644 --- a/typescript/infra/config/environments/testnet3/index.ts +++ b/typescript/infra/config/environments/testnet3/index.ts @@ -1,4 +1,4 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; +import { RpcConsensusType } from '@hyperlane-xyz/sdk'; import { getKeysForRole, @@ -27,7 +27,7 @@ export const environment: EnvironmentConfig = { getMultiProvider: ( context: Contexts = Contexts.Hyperlane, role: Role = Role.Deployer, - connectionType?: AgentConnectionType, + connectionType?: RpcConsensusType, ) => getMultiProviderForRole( testnetConfigs, diff --git a/typescript/infra/config/environments/testnet3/middleware.ts b/typescript/infra/config/environments/testnet3/middleware.ts index 2136324428..6d54c83d81 100644 --- a/typescript/infra/config/environments/testnet3/middleware.ts +++ b/typescript/infra/config/environments/testnet3/middleware.ts @@ -1,4 +1,4 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; +import { RpcConsensusType } from '@hyperlane-xyz/sdk'; import { LiquidityLayerRelayerConfig } from '../../../src/config/middleware'; @@ -12,5 +12,5 @@ export const liquidityLayerRelayerConfig: LiquidityLayerRelayerConfig = { namespace: environment, prometheusPushGateway: 'http://prometheus-pushgateway.monitoring.svc.cluster.local:9091', - connectionType: AgentConnectionType.Http, + connectionType: RpcConsensusType.Single, }; diff --git a/typescript/infra/helm/helloworld-kathy/templates/external-secret.yaml b/typescript/infra/helm/helloworld-kathy/templates/external-secret.yaml index 28af4157ec..f3d1ed0456 100644 --- a/typescript/infra/helm/helloworld-kathy/templates/external-secret.yaml +++ b/typescript/infra/helm/helloworld-kathy/templates/external-secret.yaml @@ -28,7 +28,7 @@ spec: * to replace the correct value in the created secret. */}} {{- range .Values.hyperlane.chains }} - {{- if or (eq $.Values.hyperlane.connectionType "httpQuorum") (eq $.Values.hyperlane.connectionType "httpFallback") }} + {{- if or (eq $.Values.hyperlane.connectionType "quorum") (eq $.Values.hyperlane.connectionType "fallback") }} GCP_SECRET_OVERRIDE_{{ $.Values.hyperlane.runEnv | upper }}_RPC_ENDPOINTS_{{ . | upper }}: {{ printf "'{{ .%s_rpcs | toString }}'" . }} {{- else }} GCP_SECRET_OVERRIDE_{{ $.Values.hyperlane.runEnv | upper }}_RPC_ENDPOINT_{{ . | upper }}: {{ printf "'{{ .%s_rpc | toString }}'" . }} @@ -49,7 +49,7 @@ spec: * and associate it with the secret key networkname_rpc. */}} {{- range .Values.hyperlane.chains }} - {{- if or (eq $.Values.hyperlane.connectionType "httpQuorum") (eq $.Values.hyperlane.connectionType "httpFallback") }} + {{- if or (eq $.Values.hyperlane.connectionType "quorum") (eq $.Values.hyperlane.connectionType "fallback") }} - secretKey: {{ printf "%s_rpcs" . }} remoteRef: key: {{ printf "%s-rpc-endpoints-%s" $.Values.hyperlane.runEnv . }} diff --git a/typescript/infra/helm/key-funder/templates/env-var-external-secret.yaml b/typescript/infra/helm/key-funder/templates/env-var-external-secret.yaml index 4573bd402b..6e939c5df9 100644 --- a/typescript/infra/helm/key-funder/templates/env-var-external-secret.yaml +++ b/typescript/infra/helm/key-funder/templates/env-var-external-secret.yaml @@ -28,7 +28,7 @@ spec: * to replace the correct value in the created secret. */}} {{- range .Values.hyperlane.chains }} - {{- if eq $.Values.hyperlane.connectionType "httpQuorum" }} + {{- if eq $.Values.hyperlane.connectionType "quorum" }} GCP_SECRET_OVERRIDE_{{ $.Values.hyperlane.runEnv | upper }}_RPC_ENDPOINTS_{{ . | upper }}: {{ printf "'{{ .%s_rpcs | toString }}'" . }} {{- else }} GCP_SECRET_OVERRIDE_{{ $.Values.hyperlane.runEnv | upper }}_RPC_ENDPOINT_{{ . | upper }}: {{ printf "'{{ .%s_rpc | toString }}'" . }} @@ -43,7 +43,7 @@ spec: * and associate it with the secret key networkname_rpc. */}} {{- range .Values.hyperlane.chains }} - {{- if eq $.Values.hyperlane.connectionType "httpQuorum" }} + {{- if eq $.Values.hyperlane.connectionType "quorum" }} - secretKey: {{ printf "%s_rpcs" . }} remoteRef: key: {{ printf "%s-rpc-endpoints-%s" $.Values.hyperlane.runEnv . }} diff --git a/typescript/infra/helm/liquidity-layer-relayers/templates/env-var-external-secret.yaml b/typescript/infra/helm/liquidity-layer-relayers/templates/env-var-external-secret.yaml index fd302ebfb7..1ac51df9e4 100644 --- a/typescript/infra/helm/liquidity-layer-relayers/templates/env-var-external-secret.yaml +++ b/typescript/infra/helm/liquidity-layer-relayers/templates/env-var-external-secret.yaml @@ -28,7 +28,7 @@ spec: * to replace the correct value in the created secret. */}} {{- range .Values.hyperlane.chains }} - {{- if eq $.Values.hyperlane.connectionType "httpQuorum" }} + {{- if eq $.Values.hyperlane.connectionType "quorum" }} GCP_SECRET_OVERRIDE_{{ $.Values.hyperlane.runEnv | upper }}_RPC_ENDPOINTS_{{ . | upper }}: {{ printf "'{{ .%s_rpcs | toString }}'" . }} {{- else }} GCP_SECRET_OVERRIDE_{{ $.Values.hyperlane.runEnv | upper }}_RPC_ENDPOINT_{{ . | upper }}: {{ printf "'{{ .%s_rpc | toString }}'" . }} @@ -43,7 +43,7 @@ spec: * and associate it with the secret key networkname_rpc. */}} {{- range .Values.hyperlane.chains }} - {{- if eq $.Values.hyperlane.connectionType "httpQuorum" }} + {{- if eq $.Values.hyperlane.connectionType "quorum" }} - secretKey: {{ printf "%s_rpcs" . }} remoteRef: key: {{ printf "%s-rpc-endpoints-%s" $.Values.hyperlane.runEnv . }} diff --git a/typescript/infra/scripts/agents/utils.ts b/typescript/infra/scripts/agents/utils.ts index a1d748eb28..dbae3b889b 100644 --- a/typescript/infra/scripts/agents/utils.ts +++ b/typescript/infra/scripts/agents/utils.ts @@ -48,6 +48,14 @@ export class AgentCli { } } + if (this.dryRun) { + for (const m of Object.values(managers)) { + void m.helmValues().then((v) => { + console.log(JSON.stringify(v, null, 2)); + }); + } + } + await Promise.all( Object.values(managers).map((m) => m.runHelmCommand(command, this.dryRun), diff --git a/typescript/infra/scripts/funding/fund-keys-from-deployer.ts b/typescript/infra/scripts/funding/fund-keys-from-deployer.ts index 9c25e5a56a..c93ca3057e 100644 --- a/typescript/infra/scripts/funding/fund-keys-from-deployer.ts +++ b/typescript/infra/scripts/funding/fund-keys-from-deployer.ts @@ -4,13 +4,13 @@ import { Gauge, Registry } from 'prom-client'; import { format } from 'util'; import { - AgentConnectionType, AllChains, ChainMap, ChainName, Chains, HyperlaneIgp, MultiProvider, + RpcConsensusType, } from '@hyperlane-xyz/sdk'; import { error, log, warn } from '@hyperlane-xyz/utils'; @@ -191,10 +191,10 @@ async function main() { .string('connection-type') .describe('connection-type', 'The provider connection type to use for RPCs') - .default('connection-type', AgentConnectionType.Http) + .default('connection-type', RpcConsensusType.Single) .choices('connection-type', [ - AgentConnectionType.Http, - AgentConnectionType.HttpQuorum, + RpcConsensusType.Single, + RpcConsensusType.Quorum, ]) .demandOption('connection-type') diff --git a/typescript/infra/scripts/helloworld/kathy.ts b/typescript/infra/scripts/helloworld/kathy.ts index cdd952f111..7cbc208d4b 100644 --- a/typescript/infra/scripts/helloworld/kathy.ts +++ b/typescript/infra/scripts/helloworld/kathy.ts @@ -5,13 +5,13 @@ import { format } from 'util'; import { HelloMultiProtocolApp } from '@hyperlane-xyz/helloworld'; import { - AgentConnectionType, ChainMap, ChainName, HyperlaneIgp, MultiProtocolCore, MultiProvider, ProviderType, + RpcConsensusType, TypedTransactionReceipt, chainMetadata, } from '@hyperlane-xyz/sdk'; @@ -125,11 +125,11 @@ function getKathyArgs() { .string('connection-type') .describe('connection-type', 'The provider connection type to use for RPCs') - .default('connection-type', AgentConnectionType.Http) + .default('connection-type', RpcConsensusType.Single) .choices('connection-type', [ - AgentConnectionType.Http, - AgentConnectionType.HttpQuorum, - AgentConnectionType.HttpFallback, + RpcConsensusType.Single, + RpcConsensusType.Quorum, + RpcConsensusType.Fallback, ]) .demandOption('connection-type') diff --git a/typescript/infra/scripts/helloworld/utils.ts b/typescript/infra/scripts/helloworld/utils.ts index 63276048a9..e0a45ae0ba 100644 --- a/typescript/infra/scripts/helloworld/utils.ts +++ b/typescript/infra/scripts/helloworld/utils.ts @@ -4,12 +4,12 @@ import { helloWorldFactories, } from '@hyperlane-xyz/helloworld'; import { - AgentConnectionType, HyperlaneCore, HyperlaneIgp, MultiProtocolCore, MultiProtocolProvider, MultiProvider, + RpcConsensusType, attachContractsMap, chainMetadata, filterAddressesToProtocol, @@ -30,7 +30,7 @@ export async function getHelloWorldApp( context: Contexts, keyRole: Role, keyContext: Contexts = context, - connectionType: AgentConnectionType = AgentConnectionType.Http, + connectionType: RpcConsensusType = RpcConsensusType.Single, ) { const multiProvider: MultiProvider = await coreConfig.getMultiProvider( keyContext, @@ -54,7 +54,7 @@ export async function getHelloWorldMultiProtocolApp( context: Contexts, keyRole: Role, keyContext: Contexts = context, - connectionType: AgentConnectionType = AgentConnectionType.Http, + connectionType: RpcConsensusType = RpcConsensusType.Single, ) { const multiProvider: MultiProvider = await coreConfig.getMultiProvider( keyContext, diff --git a/typescript/infra/scripts/utils.ts b/typescript/infra/scripts/utils.ts index 136324e0cb..c5f89d4b09 100644 --- a/typescript/infra/scripts/utils.ts +++ b/typescript/infra/scripts/utils.ts @@ -4,7 +4,6 @@ import path from 'path'; import yargs from 'yargs'; import { - AgentConnectionType, AllChains, ChainMap, ChainMetadata, @@ -17,6 +16,7 @@ import { MultiProvider, ProxiedRouterConfig, RouterConfig, + RpcConsensusType, collectValidators, } from '@hyperlane-xyz/sdk'; import { @@ -185,7 +185,8 @@ export async function getMultiProviderForRole( context: Contexts, role: Role, index?: number, - connectionType?: AgentConnectionType, + // TODO: rename to consensusType? + connectionType?: RpcConsensusType, ): Promise { if (process.env.CI === 'true') { return new MultiProvider(); // use default RPCs diff --git a/typescript/infra/src/agents/aws/key.ts b/typescript/infra/src/agents/aws/key.ts index b8a93a4086..fb42d10aab 100644 --- a/typescript/infra/src/agents/aws/key.ts +++ b/typescript/infra/src/agents/aws/key.ts @@ -18,9 +18,9 @@ import { import { KmsEthersSigner } from 'aws-kms-ethers-signer'; import { ethers } from 'ethers'; -import { ChainName } from '@hyperlane-xyz/sdk'; +import { AgentSignerKeyType, ChainName } from '@hyperlane-xyz/sdk'; -import { AgentContextConfig, AwsKeyConfig, KeyType } from '../../config/agent'; +import { AgentContextConfig, AwsKeyConfig } from '../../config/agent'; import { Role } from '../../roles'; import { getEthereumAddress, sleep } from '../../utils/utils'; import { keyIdentifier } from '../agent'; @@ -81,7 +81,7 @@ export class AgentAwsKey extends CloudAgentKey { get keyConfig(): AwsKeyConfig { return { - type: KeyType.Aws, + type: AgentSignerKeyType.Aws, id: this.identifier, region: this.region, }; diff --git a/typescript/infra/src/agents/index.ts b/typescript/infra/src/agents/index.ts index ef73f414f5..1d306963cc 100644 --- a/typescript/infra/src/agents/index.ts +++ b/typescript/infra/src/agents/index.ts @@ -1,10 +1,6 @@ import fs from 'fs'; -import { - AgentConnectionType, - ChainName, - chainMetadata, -} from '@hyperlane-xyz/sdk'; +import { ChainName, RpcConsensusType, chainMetadata } from '@hyperlane-xyz/sdk'; import { ProtocolType } from '@hyperlane-xyz/utils'; import { Contexts } from '../../config/contexts'; @@ -120,18 +116,18 @@ export abstract class AgentHelmManager { chains: this.config.environmentChainNames.map((name) => ({ name, disabled: !this.config.contextChainNames[this.role].includes(name), - connection: { type: this.connectionType(name) }, + rpcConsensusType: this.rpcConsensusType(name), })), }, }; } - connectionType(chain: ChainName): AgentConnectionType { + rpcConsensusType(chain: ChainName): RpcConsensusType { if (chainMetadata[chain].protocol == ProtocolType.Sealevel) { - return AgentConnectionType.Http; + return RpcConsensusType.Single; } - return this.config.connectionType; + return this.config.rpcConsensusType; } async doesAgentReleaseExist() { @@ -252,9 +248,20 @@ export class ValidatorHelmManager extends MultichainAgentHelmManager { async helmValues(): Promise { const helmValues = await super.helmValues(); + const cfg = await this.config.buildConfig(); + + helmValues.hyperlane.chains.push({ + name: cfg.originChainName, + blocks: { reorgPeriod: cfg.reorgPeriod }, + }); + helmValues.hyperlane.validator = { enabled: true, - configs: await this.config.buildConfig(), + configs: cfg.validators.map((c) => ({ + ...c, + originChainName: cfg.originChainName, + interval: cfg.interval, + })), }; // The name of the helm release for agents is `hyperlane-agent`. diff --git a/typescript/infra/src/config/agent/agent.ts b/typescript/infra/src/config/agent/agent.ts index 7ad77fc10e..af11ea850c 100644 --- a/typescript/infra/src/config/agent/agent.ts +++ b/typescript/infra/src/config/agent/agent.ts @@ -1,8 +1,9 @@ import { - AgentChainSetup, - AgentConnection, - AgentConnectionType, + AgentChainMetadata, + AgentSignerAwsKey, + AgentSignerKeyType, ChainName, + RpcConsensusType, } from '@hyperlane-xyz/sdk'; import { Contexts } from '../../../config/contexts'; @@ -18,6 +19,12 @@ import { import { BaseScraperConfig, HelmScraperValues } from './scraper'; import { HelmValidatorValues, ValidatorBaseChainConfigMap } from './validator'; +export type DeepPartial = T extends object + ? { + [P in keyof T]?: DeepPartial; + } + : T; + // See rust/helm/values.yaml for the full list of options and their defaults. // This is the root object in the values file. export interface HelmRootAgentValues { @@ -45,10 +52,9 @@ interface HelmHyperlaneValues { // See rust/helm/values.yaml for the full list of options and their defaults. // This is at `.hyperlane.chains` in the values file. export interface HelmAgentChainOverride - extends Partial> { - name: ChainName; + extends DeepPartial { + name: AgentChainMetadata['name']; disabled?: boolean; - connection?: Partial; } export interface RootAgentConfig extends AgentContextConfig { @@ -80,29 +86,14 @@ export interface AgentContextConfig extends AgentEnvConfig { interface AgentRoleConfig { docker: DockerConfig; chainDockerOverrides?: Record>; - quorumProvider?: boolean; - connectionType: AgentConnectionType; + rpcConsensusType: RpcConsensusType; index?: IndexingConfig; } -export enum KeyType { - Aws = 'aws', - Hex = 'hexKey', -} - -export interface AwsKeyConfig { - type: KeyType.Aws; - // ID of the key, can be an alias of the form `alias/foo-bar` - id: string; - // AWS region where the key is - region: string; -} - -// The private key is omitted so it can be fetched using external-secrets -export interface HexKeyConfig { - type: KeyType.Hex; -} - +// require specifying that it's the "aws" type for helm +export type AwsKeyConfig = Required; +// only require specifying that it's the "hex" type for helm since the hex key will be pulled from secrets. +export type HexKeyConfig = { type: AgentSignerKeyType.Hex }; export type KeyConfig = AwsKeyConfig | HexKeyConfig; interface IndexingConfig { @@ -158,14 +149,14 @@ export abstract class AgentConfigHelper extends RootAgentConfigHelper implements AgentRoleConfig { - connectionType: AgentConnectionType; + rpcConsensusType: RpcConsensusType; docker: DockerConfig; chainDockerOverrides?: Record>; index?: IndexingConfig; protected constructor(root: RootAgentConfig, agent: AgentRoleConfig) { super(root); - this.connectionType = agent.connectionType; + this.rpcConsensusType = agent.rpcConsensusType; this.docker = agent.docker; this.chainDockerOverrides = agent.chainDockerOverrides; this.index = agent.index; diff --git a/typescript/infra/src/config/agent/index.ts b/typescript/infra/src/config/agent/index.ts index 5330637b85..802717a735 100644 --- a/typescript/infra/src/config/agent/index.ts +++ b/typescript/infra/src/config/agent/index.ts @@ -5,11 +5,7 @@ export { CheckpointSyncerType, ValidatorBaseChainConfigMap, } from './validator'; -export { - RelayerConfigHelper, - GasPaymentEnforcementPolicyType, - routerMatchingList, -} from './relayer'; +export { RelayerConfigHelper, routerMatchingList } from './relayer'; export { ScraperConfigHelper } from './scraper'; export * from './agent'; diff --git a/typescript/infra/src/config/agent/relayer.ts b/typescript/infra/src/config/agent/relayer.ts index c778989750..f68f37c155 100644 --- a/typescript/infra/src/config/agent/relayer.ts +++ b/typescript/infra/src/config/agent/relayer.ts @@ -1,78 +1,35 @@ import { BigNumberish } from 'ethers'; -import { ChainMap, chainMetadata } from '@hyperlane-xyz/sdk'; +import { + AgentConfig, + AgentSignerKeyType, + ChainMap, + MatchingList, + chainMetadata, +} from '@hyperlane-xyz/sdk'; +import { GasPaymentEnforcement } from '@hyperlane-xyz/sdk'; +import { RelayerConfig as RelayerAgentConfig } from '@hyperlane-xyz/sdk'; import { ProtocolType } from '@hyperlane-xyz/utils'; import { AgentAwsUser } from '../../agents/aws'; import { Role } from '../../roles'; import { HelmStatefulSetValues } from '../infrastructure'; -import { - AgentConfigHelper, - KeyConfig, - KeyType, - RootAgentConfig, -} from './agent'; - -export type MatchingList = MatchingListElement[]; - -export interface MatchingListElement { - originDomain?: '*' | number | number[]; - senderAddress?: '*' | string | string[]; - destinationDomain?: '*' | number | number[]; - recipientAddress?: '*' | string | string[]; -} +import { AgentConfigHelper, KeyConfig, RootAgentConfig } from './agent'; -export enum GasPaymentEnforcementPolicyType { - None = 'none', - Minimum = 'minimum', - MeetsEstimatedCost = 'meetsEstimatedCost', - OnChainFeeQuoting = 'onChainFeeQuoting', -} - -export type GasPaymentEnforcementPolicy = - | { - type: GasPaymentEnforcementPolicyType.None; - } - | { - type: GasPaymentEnforcementPolicyType.Minimum; - payment: string; // An integer string, may be 0x-prefixed - } - | { - type: GasPaymentEnforcementPolicyType.OnChainFeeQuoting; - gasfraction?: string; // An optional string of "numerator / denominator", e.g. "1 / 2" - }; - -export type GasPaymentEnforcementConfig = GasPaymentEnforcementPolicy & { - matchingList?: MatchingList; -}; +export { GasPaymentEnforcement as GasPaymentEnforcementConfig } from '@hyperlane-xyz/sdk'; // Incomplete basic relayer agent config export interface BaseRelayerConfig { - gasPaymentEnforcement: GasPaymentEnforcementConfig[]; + gasPaymentEnforcement: GasPaymentEnforcement[]; whitelist?: MatchingList; blacklist?: MatchingList; transactionGasLimit?: BigNumberish; - skipTransactionGasLimitFor?: number[]; + skipTransactionGasLimitFor?: string[]; } -// Full relayer agent config for a single chain -export interface RelayerConfig - extends Omit< - BaseRelayerConfig, - | 'whitelist' - | 'blacklist' - | 'skipTransactionGasLimitFor' - | 'transactionGasLimit' - | 'gasPaymentEnforcement' - > { - relayChains: string; - gasPaymentEnforcement: string; - whitelist?: string; - blacklist?: string; - transactionGasLimit?: string; - skipTransactionGasLimitFor?: string; -} +// Full relayer-specific agent config for a single chain +export type RelayerConfig = Omit; // See rust/helm/values.yaml for the full list of options and their defaults. // This is at `.hyperlane.relayer` in the values file. @@ -140,7 +97,7 @@ export class RelayerConfigHelper extends AgentConfigHelper { const chain = chainMetadata[name]; // Sealevel chains always use hex keys if (chain?.protocol == ProtocolType.Sealevel) { - return [name, { type: KeyType.Hex }]; + return [name, { type: AgentSignerKeyType.Hex }]; } else { return [name, awsKey]; } @@ -150,7 +107,7 @@ export class RelayerConfigHelper extends AgentConfigHelper { return Object.fromEntries( this.contextChainNames[Role.Relayer].map((name) => [ name, - { type: KeyType.Hex }, + { type: AgentSignerKeyType.Hex }, ]), ); } @@ -175,9 +132,12 @@ export class RelayerConfigHelper extends AgentConfigHelper { } // Create a matching list for the given router addresses -export function routerMatchingList(routers: ChainMap<{ router: string }>) { +export function routerMatchingList( + routers: ChainMap<{ router: string }>, +): MatchingList { const chains = Object.keys(routers); + // matching list must have at least one element so bypass and check before returning const matchingList: MatchingList = []; for (const source of chains) { @@ -194,5 +154,6 @@ export function routerMatchingList(routers: ChainMap<{ router: string }>) { }); } } + return matchingList; } diff --git a/typescript/infra/src/config/agent/scraper.ts b/typescript/infra/src/config/agent/scraper.ts index 3379e3b664..6bb35dde03 100644 --- a/typescript/infra/src/config/agent/scraper.ts +++ b/typescript/infra/src/config/agent/scraper.ts @@ -1,3 +1,8 @@ +import { + AgentConfig, + ScraperConfig as ScraperAgentConfig, +} from '@hyperlane-xyz/sdk'; + import { Role } from '../../roles'; import { HelmStatefulSetValues } from '../infrastructure'; @@ -8,7 +13,8 @@ export interface BaseScraperConfig { __placeholder?: undefined; } -export type ScraperConfig = BaseScraperConfig; +// Ignore db which is added by helm +export type ScraperConfig = Omit; export interface HelmScraperValues extends HelmStatefulSetValues { config?: ScraperConfig; @@ -22,7 +28,9 @@ export class ScraperConfigHelper extends AgentConfigHelper { } async buildConfig(): Promise { - return {}; + return { + chainsToScrape: this.contextChainNames[Role.Scraper].join(','), + }; } get role(): Role { diff --git a/typescript/infra/src/config/agent/validator.ts b/typescript/infra/src/config/agent/validator.ts index 36d5c89022..380ebef111 100644 --- a/typescript/infra/src/config/agent/validator.ts +++ b/typescript/infra/src/config/agent/validator.ts @@ -1,15 +1,16 @@ -import { ChainMap, ChainName } from '@hyperlane-xyz/sdk'; +import { + AgentConfig, + AgentSignerKeyType, + ValidatorConfig as AgentValidatorConfig, + ChainMap, + ChainName, +} from '@hyperlane-xyz/sdk'; import { ValidatorAgentAwsUser } from '../../agents/aws'; import { Role } from '../../roles'; import { HelmStatefulSetValues } from '../infrastructure'; -import { - AgentConfigHelper, - KeyConfig, - KeyType, - RootAgentConfig, -} from './agent'; +import { AgentConfigHelper, KeyConfig, RootAgentConfig } from './agent'; // Validator agents for each chain. export type ValidatorBaseChainConfigMap = ChainMap; @@ -17,7 +18,7 @@ export type ValidatorBaseChainConfigMap = ChainMap; export interface ValidatorBaseChainConfig { // How frequently to check for new checkpoints interval: number; - // The reorg_period in blocks + // The reorg_period in blocks; overrides chain metadata reorgPeriod: number; // Individual validator agents validators: Array; @@ -30,17 +31,24 @@ export interface ValidatorBaseConfig { checkpointSyncer: CheckpointSyncerConfig; } -// Full config for a single validator export interface ValidatorConfig { interval: number; reorgPeriod: number; originChainName: ChainName; - checkpointSyncer: CheckpointSyncerConfig; - validator: KeyConfig; + validators: Array<{ + checkpointSyncer: CheckpointSyncerConfig; + validator: KeyConfig; + }>; } export interface HelmValidatorValues extends HelmStatefulSetValues { - configs?: ValidatorConfig[]; + configs?: Array< + // only keep configs specific to the validator agent and then replace + // the validator signing key with the version helm needs. + Omit & { + validator: KeyConfig; + } + >; } export type CheckpointSyncerConfig = @@ -64,9 +72,7 @@ export interface S3CheckpointSyncerConfig { region: string; } -export class ValidatorConfigHelper extends AgentConfigHelper< - Array -> { +export class ValidatorConfigHelper extends AgentConfigHelper { readonly #validatorsConfig: ValidatorBaseChainConfigMap; constructor( @@ -79,12 +85,17 @@ export class ValidatorConfigHelper extends AgentConfigHelper< this.#validatorsConfig = agentConfig.validators.chains; } - async buildConfig(): Promise> { - return Promise.all( - this.#chainConfig.validators.map(async (val, i) => - this.#configForValidator(val, i), + async buildConfig(): Promise { + return { + interval: this.#chainConfig.interval, + reorgPeriod: this.#chainConfig.reorgPeriod, + originChainName: this.chainName!, + validators: await Promise.all( + this.#chainConfig.validators.map((val, i) => + this.#configForValidator(val, i), + ), ), - ); + }; } get validators(): ValidatorBaseConfig[] { @@ -98,8 +109,8 @@ export class ValidatorConfigHelper extends AgentConfigHelper< async #configForValidator( cfg: ValidatorBaseConfig, idx: number, - ): Promise { - let validator: KeyConfig = { type: KeyType.Hex }; + ): Promise { + let validator: KeyConfig = { type: AgentSignerKeyType.Hex }; if (cfg.checkpointSyncer.type == CheckpointSyncerType.S3) { const awsUser = new ValidatorAgentAwsUser( this.runEnv, @@ -121,10 +132,7 @@ export class ValidatorConfigHelper extends AgentConfigHelper< } return { - interval: this.#chainConfig.interval, - reorgPeriod: this.#chainConfig.reorgPeriod, checkpointSyncer: cfg.checkpointSyncer, - originChainName: this.chainName!, validator, }; } diff --git a/typescript/infra/src/config/chain.ts b/typescript/infra/src/config/chain.ts index 617be5512f..fee1568ce1 100644 --- a/typescript/infra/src/config/chain.ts +++ b/typescript/infra/src/config/chain.ts @@ -1,10 +1,10 @@ import { providers } from 'ethers'; import { - AgentConnectionType, ChainName, RetryJsonRpcProvider, RetryProviderOptions, + RpcConsensusType, } from '@hyperlane-xyz/sdk'; import { getSecretRpcEndpoint } from '../agents'; @@ -29,20 +29,20 @@ function buildProvider(config?: { export async function fetchProvider( environment: DeployEnvironment, chainName: ChainName, - connectionType: AgentConnectionType = AgentConnectionType.Http, + connectionType: RpcConsensusType = RpcConsensusType.Single, ): Promise { - const single = connectionType === AgentConnectionType.Http; + const single = connectionType === RpcConsensusType.Single; const rpcData = await getSecretRpcEndpoint(environment, chainName, !single); switch (connectionType) { - case AgentConnectionType.Http: { + case RpcConsensusType.Single: { return buildProvider({ url: rpcData[0], retry: defaultRetry }); } - case AgentConnectionType.HttpQuorum: { + case RpcConsensusType.Quorum: { return new providers.FallbackProvider( (rpcData as string[]).map((url) => buildProvider({ url })), // disable retry for quorum ); } - case AgentConnectionType.HttpFallback: { + case RpcConsensusType.Fallback: { return new providers.FallbackProvider( (rpcData as string[]).map((url, index) => { const fallbackProviderConfig: providers.FallbackProviderConfig = { diff --git a/typescript/infra/src/config/environment.ts b/typescript/infra/src/config/environment.ts index f3e8f401be..ec3ebd7b14 100644 --- a/typescript/infra/src/config/environment.ts +++ b/typescript/infra/src/config/environment.ts @@ -1,5 +1,4 @@ import { - AgentConnectionType, BridgeAdapterConfig, ChainMap, ChainMetadata, @@ -9,6 +8,7 @@ import { InterceptorConfig, MultiProvider, OverheadIgpConfig, + RpcConsensusType, } from '@hyperlane-xyz/sdk'; import { Address } from '@hyperlane-xyz/utils'; @@ -44,7 +44,7 @@ export type EnvironmentConfig = { getMultiProvider: ( context?: Contexts, role?: Role, - connectionType?: AgentConnectionType, + connectionType?: RpcConsensusType, ) => Promise; getKeys: ( context?: Contexts, diff --git a/typescript/infra/src/config/funding.ts b/typescript/infra/src/config/funding.ts index 74738a2375..3ad6f48aa0 100644 --- a/typescript/infra/src/config/funding.ts +++ b/typescript/infra/src/config/funding.ts @@ -1,4 +1,4 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; +import { RpcConsensusType } from '@hyperlane-xyz/sdk'; import { Contexts } from '../../config/contexts'; import { Role } from '../roles'; @@ -20,5 +20,5 @@ export interface KeyFunderConfig { contextsAndRolesToFund: ContextAndRolesMap; cyclesBetweenEthereumMessages?: number; prometheusPushGateway: string; - connectionType: AgentConnectionType.Http | AgentConnectionType.HttpQuorum; + connectionType: RpcConsensusType.Single | RpcConsensusType.Quorum; } diff --git a/typescript/infra/src/config/helloworld.ts b/typescript/infra/src/config/helloworld.ts index 071e7464f3..3f9d97700b 100644 --- a/typescript/infra/src/config/helloworld.ts +++ b/typescript/infra/src/config/helloworld.ts @@ -1,4 +1,4 @@ -import { AgentConnectionType, ChainMap, ChainName } from '@hyperlane-xyz/sdk'; +import { ChainMap, ChainName, RpcConsensusType } from '@hyperlane-xyz/sdk'; import { DockerConfig } from './agent'; @@ -29,7 +29,7 @@ export interface HelloWorldKathyConfig { messageReceiptTimeout: number; // Which type of provider to use - connectionType: Exclude; + connectionType: RpcConsensusType; // How many cycles to skip between a cycles that send messages to/from Ethereum. Defaults to 0. cyclesBetweenEthereumMessages?: number; } diff --git a/typescript/infra/src/config/middleware.ts b/typescript/infra/src/config/middleware.ts index 052a4360ee..907125b700 100644 --- a/typescript/infra/src/config/middleware.ts +++ b/typescript/infra/src/config/middleware.ts @@ -1,10 +1,10 @@ -import { AgentConnectionType } from '@hyperlane-xyz/sdk'; +import { RpcConsensusType } from '@hyperlane-xyz/sdk'; import { DockerConfig } from './agent'; export interface LiquidityLayerRelayerConfig { docker: DockerConfig; namespace: string; - connectionType: AgentConnectionType.Http | AgentConnectionType.HttpQuorum; + connectionType: RpcConsensusType.Single | RpcConsensusType.Quorum; prometheusPushGateway: string; } diff --git a/typescript/infra/src/deployment/deploy.ts b/typescript/infra/src/deployment/deploy.ts index 71baf348f9..01a90875d3 100644 --- a/typescript/infra/src/deployment/deploy.ts +++ b/typescript/infra/src/deployment/deploy.ts @@ -5,7 +5,7 @@ import { HyperlaneDeployer, HyperlaneDeploymentArtifacts, MultiProvider, - buildAgentConfigDeprecated, + buildAgentConfig, serializeContractsMap, } from '@hyperlane-xyz/sdk'; import { objMap, objMerge, promiseObjAll } from '@hyperlane-xyz/utils'; @@ -137,7 +137,7 @@ export async function writeAgentConfig( multiProvider.getProvider(chain).getBlockNumber(), ), ); - const agentConfig = buildAgentConfigDeprecated( + const agentConfig = buildAgentConfig( multiProvider.getKnownChainNames(), multiProvider, addresses as ChainMap, diff --git a/typescript/infra/tsconfig.json b/typescript/infra/tsconfig.json index ae3a07ae39..13290d01d9 100644 --- a/typescript/infra/tsconfig.json +++ b/typescript/infra/tsconfig.json @@ -5,7 +5,7 @@ "noUnusedLocals": false, }, "exclude": ["./node_modules/", "./dist/", "./tmp.ts"], - "extends": "../../tsconfig.json", + "extends": "../tsconfig.json", "include": [ "./*.ts", "./config/**/*.ts", diff --git a/typescript/sdk/src/index.ts b/typescript/sdk/src/index.ts index 48e92c22d7..587ae8b4b9 100644 --- a/typescript/sdk/src/index.ts +++ b/typescript/sdk/src/index.ts @@ -138,22 +138,24 @@ export { export { AgentChainMetadata, AgentChainMetadataSchema, - AgentChainSetup, - AgentChainSetupBase, AgentConfig, AgentConfigSchema, - AgentConfigV2, - AgentConnection, - AgentConnectionType, AgentLogFormat, AgentLogLevel, AgentSigner, - AgentSignerSchema, - AgentSignerV2, + AgentSignerKeyType, + AgentSignerHexKey, + AgentSignerAwsKey, + AgentSignerNode, buildAgentConfig, - buildAgentConfigDeprecated, - buildAgentConfigNew, + RpcConsensusType, + ValidatorConfig, + GasPaymentEnforcement, + RelayerConfig, + GasPaymentEnforcementPolicyType, + ScraperConfig, } from './metadata/agentConfig'; +export { MatchingList } from './metadata/matchingList'; export { ChainMetadata, ChainMetadataSchema, diff --git a/typescript/sdk/src/metadata/agentConfig.test.ts b/typescript/sdk/src/metadata/agentConfig.test.ts index 4f151d374e..264473ac48 100644 --- a/typescript/sdk/src/metadata/agentConfig.test.ts +++ b/typescript/sdk/src/metadata/agentConfig.test.ts @@ -3,11 +3,7 @@ import { expect } from 'chai'; import { Chains } from '../consts/chains'; import { MultiProvider } from '../providers/MultiProvider'; -import { - buildAgentConfig, - buildAgentConfigDeprecated, - buildAgentConfigNew, -} from './agentConfig'; +import { buildAgentConfig } from './agentConfig'; describe('Agent config', () => { const args: Parameters = [ @@ -23,18 +19,18 @@ describe('Agent config', () => { { ethereum: 0 }, ]; - it('Should generate a deprecated agent config', () => { - const result = buildAgentConfigDeprecated(...args); - expect(Object.keys(result)).to.deep.equal(['chains']); - }); - it('Should generate a new agent config', () => { - const result = buildAgentConfigNew(...args); - expect(Object.keys(result)).to.deep.equal([Chains.ethereum]); - }); - - it('Should generate a combined agent config', () => { const result = buildAgentConfig(...args); - expect(Object.keys(result)).to.deep.equal([Chains.ethereum, 'chains']); + expect(Object.keys(result)).to.deep.equal([ + 'chains', + 'defaultRpcConsensusType', + ]); + expect(result.chains[Chains.ethereum].mailbox).to.equal('0xmailbox'); + expect(result.chains[Chains.ethereum].interchainGasPaymaster).to.equal( + '0xgas', + ); + expect(result.chains[Chains.ethereum].validatorAnnounce).to.equal( + '0xannounce', + ); }); }); diff --git a/typescript/sdk/src/metadata/agentConfig.ts b/typescript/sdk/src/metadata/agentConfig.ts index 1ed8357ba7..30c0096a4b 100644 --- a/typescript/sdk/src/metadata/agentConfig.ts +++ b/typescript/sdk/src/metadata/agentConfig.ts @@ -4,16 +4,10 @@ */ import { z } from 'zod'; -import { ProtocolType } from '@hyperlane-xyz/utils'; - import { MultiProvider } from '../providers/MultiProvider'; import { ChainMap, ChainName } from '../types'; -import { - ChainMetadata, - ChainMetadataSchema, - RpcUrlSchema, -} from './chainMetadataTypes'; +import { ChainMetadata, ChainMetadataSchema } from './chainMetadataTypes'; import { ZHash, ZNzUint, ZUWei, ZUint } from './customZodTypes'; import { HyperlaneDeploymentArtifacts, @@ -21,14 +15,8 @@ import { } from './deploymentArtifacts'; import { MatchingListSchema } from './matchingList'; -export enum AgentConnectionType { - Http = 'http', - Ws = 'ws', - HttpQuorum = 'httpQuorum', - HttpFallback = 'httpFallback', -} - -export enum AgentConsensusType { +export enum RpcConsensusType { + Single = 'single', Fallback = 'fallback', Quorum = 'quorum', } @@ -54,52 +42,55 @@ export enum AgentIndexMode { Sequence = 'sequence', } -export const AgentSignerSchema = z.union([ - z - .object({ - type: z.literal('hexKey').optional(), - key: ZHash, - }) - .describe('A local hex key'), - z - .object({ - type: z.literal('aws').optional(), - id: z.string().describe('The UUID identifying the AWS KMS key'), - region: z.string().describe('The AWS region'), - }) - .describe( - 'An AWS signer. Note that AWS credentials must be inserted into the env separately.', - ), - z - .object({ - type: z.literal('node'), - }) - .describe('Assume the local node will sign on RPC calls automatically'), +export enum AgentSignerKeyType { + Aws = 'aws', + Hex = 'hexKey', + Node = 'node', +} + +const AgentSignerHexKeySchema = z + .object({ + type: z.literal(AgentSignerKeyType.Hex).optional(), + key: ZHash, + }) + .describe('A local hex key'); +const AgentSignerAwsKeySchema = z + .object({ + type: z.literal(AgentSignerKeyType.Aws).optional(), + id: z.string().describe('The UUID identifying the AWS KMS key'), + region: z.string().describe('The AWS region'), + }) + .describe( + 'An AWS signer. Note that AWS credentials must be inserted into the env separately.', + ); +const AgentSignerNodeSchema = z + .object({ + type: z.literal(AgentSignerKeyType.Node), + }) + .describe('Assume the local node will sign on RPC calls automatically'); + +const AgentSignerSchema = z.union([ + AgentSignerHexKeySchema, + AgentSignerAwsKeySchema, + AgentSignerNodeSchema, ]); -export type AgentSignerV2 = z.infer; +export type AgentSignerHexKey = z.infer; +export type AgentSignerAwsKey = z.infer; +export type AgentSignerNode = z.infer; +export type AgentSigner = z.infer; export const AgentChainMetadataSchema = ChainMetadataSchema.merge( HyperlaneDeploymentArtifactsSchema, ).extend({ customRpcUrls: z - .record( - RpcUrlSchema.extend({ - priority: ZNzUint.optional().describe( - 'The priority of this RPC relative to the others defined. A larger value means it will be preferred. Only effects some AgentConsensusTypes.', - ), - }), - ) - .refine((data) => Object.keys(data).length > 0, { - message: - 'Must specify at least one RPC url if not using the default rpcUrls.', - }) + .string() .optional() .describe( - 'Specify a custom RPC endpoint configuration for this chain. If this is set, then none of the `rpcUrls` will be used for this chain. The key value can be any valid string.', + 'Specify a comma seperated list of custom RPC URLs to use for this chain. If not specified, the default RPC urls will be used.', ), rpcConsensusType: z - .nativeEnum(AgentConsensusType) + .nativeEnum(RpcConsensusType) .describe('The consensus type to use when multiple RPCs are configured.') .optional(), signer: AgentSignerSchema.optional().describe( @@ -113,7 +104,6 @@ export const AgentChainMetadataSchema = ChainMetadataSchema.merge( chunk: ZNzUint.optional().describe( 'The number of blocks to index at a time.', ), - // TODO(2214): I think we can always interpret this from the ProtocolType mode: z .nativeEnum(AgentIndexMode) .optional() @@ -149,7 +139,7 @@ export const AgentConfigSchema = z.object({ 'Default signer to use for any chains that have not defined their own.', ), defaultRpcConsensusType: z - .nativeEnum(AgentConsensusType) + .nativeEnum(RpcConsensusType) .describe( 'The default consensus type to use for any chains that have not defined their own.', ) @@ -171,30 +161,33 @@ export const AgentConfigSchema = z.object({ const CommaSeperatedChainList = z.string().regex(/^[a-z0-9]+(,[a-z0-9]+)*$/); const CommaSeperatedDomainList = z.string().regex(/^\d+(,\d+)*$/); +export enum GasPaymentEnforcementPolicyType { + None = 'none', + Minimum = 'minimum', + OnChainFeeQuoting = 'onChainFeeQuoting', +} + const GasPaymentEnforcementBaseSchema = z.object({ matchingList: MatchingListSchema.optional().describe( 'An optional matching list, any message that matches will use this policy. By default all messages will match.', ), }); -const GasPaymentEnforcementSchema = z.array( - z.union([ - GasPaymentEnforcementBaseSchema.extend({ - type: z.literal('none').optional(), - }), - GasPaymentEnforcementBaseSchema.extend({ - type: z.literal('minimum').optional(), - payment: ZUWei, - }), - GasPaymentEnforcementBaseSchema.extend({ - type: z.literal('onChainFeeQuoting'), - gasFraction: z - .string() - .regex(/^\d+ ?\/ ?[1-9]\d*$/) - .optional(), - }), - ]), -); - +const GasPaymentEnforcementSchema = z.union([ + GasPaymentEnforcementBaseSchema.extend({ + type: z.literal(GasPaymentEnforcementPolicyType.None).optional(), + }), + GasPaymentEnforcementBaseSchema.extend({ + type: z.literal(GasPaymentEnforcementPolicyType.Minimum).optional(), + payment: ZUWei, + }), + GasPaymentEnforcementBaseSchema.extend({ + type: z.literal(GasPaymentEnforcementPolicyType.OnChainFeeQuoting), + gasFraction: z + .string() + .regex(/^\d+ ?\/ ?[1-9]\d*$/) + .optional(), + }), +]); export type GasPaymentEnforcement = z.infer; export const RelayerAgentConfigSchema = AgentConfigSchema.extend({ @@ -207,7 +200,7 @@ export const RelayerAgentConfigSchema = AgentConfigSchema.extend({ 'Comma seperated list of chains to relay messages between.', ), gasPaymentEnforcement: z - .union([GasPaymentEnforcementSchema, z.string().nonempty()]) + .union([z.array(GasPaymentEnforcementSchema), z.string().nonempty()]) .optional() .describe( 'The gas payment enforcement configuration as JSON. Expects an ordered array of `GasPaymentEnforcementConfig`.', @@ -292,63 +285,18 @@ export const ValidatorAgentConfigSchema = AgentConfigSchema.extend({ export type ValidatorConfig = z.infer; -export type AgentConfigV2 = z.infer; - -/** - * Deprecated agent config shapes. - * See https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/2215 - */ - -export interface AgentSigner { - key: string; - type: string; -} - -export type AgentConnection = - | { type: AgentConnectionType.Http; url: string } - | { type: AgentConnectionType.Ws; url: string } - | { type: AgentConnectionType.HttpQuorum; urls: string } - | { type: AgentConnectionType.HttpFallback; urls: string }; - -export interface AgentChainSetupBase { - name: ChainName; - domain: number; - signer?: AgentSigner; - finalityBlocks: number; - addresses: HyperlaneDeploymentArtifacts; - protocol: ProtocolType; - connection?: AgentConnection; - index?: { from: number }; -} - -export interface AgentChainSetup extends AgentChainSetupBase { - signer: AgentSigner; - connection: AgentConnection; -} - -export interface AgentConfig { - chains: Partial>; - tracing?: { - level?: string; - fmt?: 'json'; - }; -} +export type AgentConfig = z.infer; -/** - * Utilities for generating agent configs from metadata / artifacts. - */ - -// Returns the new agent config shape that extends ChainMetadata -export function buildAgentConfigNew( +export function buildAgentConfig( chains: ChainName[], multiProvider: MultiProvider, addresses: ChainMap, startBlocks: ChainMap, -): ChainMap { - const configs: ChainMap = {}; +): AgentConfig { + const chainConfigs: ChainMap = {}; for (const chain of [...chains].sort()) { const metadata: ChainMetadata = multiProvider.getChainMetadata(chain); - const config: AgentChainMetadata = { + const chainConfig: AgentChainMetadata = { ...metadata, mailbox: addresses[chain].mailbox, interchainGasPaymaster: addresses[chain].interchainGasPaymaster, @@ -357,61 +305,11 @@ export function buildAgentConfigNew( from: startBlocks[chain], }, }; - configs[chain] = config; - } - return configs; -} - -// Returns the current (but deprecated) agent config shape. -export function buildAgentConfigDeprecated( - chains: ChainName[], - multiProvider: MultiProvider, - addresses: ChainMap, - startBlocks: ChainMap, -): AgentConfig { - const agentConfig: AgentConfig = { - chains: {}, - }; - - for (const chain of [...chains].sort()) { - const metadata = multiProvider.getChainMetadata(chain); - const chainConfig: AgentChainSetupBase = { - name: chain, - domain: metadata.chainId, - addresses: { - mailbox: addresses[chain].mailbox, - interchainGasPaymaster: addresses[chain].interchainGasPaymaster, - validatorAnnounce: addresses[chain].validatorAnnounce, - }, - protocol: metadata.protocol, - finalityBlocks: metadata.blocks?.reorgPeriod ?? 1, - }; - - chainConfig.index = { - from: startBlocks[chain], - }; - - agentConfig.chains[chain] = chainConfig; + chainConfigs[chain] = chainConfig; } - return agentConfig; -} - -// TODO(2215): this eventually needs to to be replaced with just `AgentConfig2` (and that ident needs renaming) -export type CombinedAgentConfig = AgentConfigV2['chains'] | AgentConfig; -export function buildAgentConfig( - chains: ChainName[], - multiProvider: MultiProvider, - addresses: ChainMap, - startBlocks: ChainMap, -): CombinedAgentConfig { return { - ...buildAgentConfigNew(chains, multiProvider, addresses, startBlocks), - ...buildAgentConfigDeprecated( - chains, - multiProvider, - addresses, - startBlocks, - ), + chains: chainConfigs, + defaultRpcConsensusType: RpcConsensusType.Fallback, }; } diff --git a/typescript/sdk/src/metadata/chainMetadataTypes.ts b/typescript/sdk/src/metadata/chainMetadataTypes.ts index b448bc32e3..caa9297e29 100644 --- a/typescript/sdk/src/metadata/chainMetadataTypes.ts +++ b/typescript/sdk/src/metadata/chainMetadataTypes.ts @@ -60,6 +60,12 @@ export type RpcUrl = z.infer; * Specified as a Zod schema */ export const ChainMetadataSchema = z.object({ + name: z + .string() + .regex(/^[a-z][a-z0-9]*$/) + .describe( + 'The unique string identifier of the chain, used as the key in ChainMap dictionaries.', + ), protocol: z .nativeEnum(ProtocolType) .describe( @@ -71,12 +77,6 @@ export const ChainMetadataSchema = z.object({ domainId: ZNzUint.optional().describe( 'The domainId of the chain, should generally default to `chainId`. Consumer of `ChainMetadata` should use this value if present, but otherwise fallback to `chainId`.', ), - name: z - .string() - .regex(/^[a-z][a-z0-9]*$/) - .describe( - 'The unique string identifier of the chain, used as the key in ChainMap dictionaries.', - ), displayName: z .string() .optional() diff --git a/typescript/sdk/src/metadata/matchingList.ts b/typescript/sdk/src/metadata/matchingList.ts index 5d422edde4..336c37cdeb 100644 --- a/typescript/sdk/src/metadata/matchingList.ts +++ b/typescript/sdk/src/metadata/matchingList.ts @@ -25,7 +25,7 @@ const MatchingListElementSchema = z.object({ recipientAddress: AddressSchema.optional(), }); -export const MatchingListSchema = z.array(MatchingListElementSchema).nonempty(); +export const MatchingListSchema = z.array(MatchingListElementSchema); export type MatchingListElement = z.infer; export type MatchingList = z.infer; diff --git a/typescript/sdk/tsconfig.json b/typescript/sdk/tsconfig.json index 8d537a5b6c..9bf7368a74 100644 --- a/typescript/sdk/tsconfig.json +++ b/typescript/sdk/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../tsconfig.json", "compilerOptions": { "outDir": "./dist/", "rootDir": "./src/" diff --git a/typescript/token/tsconfig.json b/typescript/token/tsconfig.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/typescript/utils/tsconfig.json b/typescript/utils/tsconfig.json index 821816744d..057d76f65c 100644 --- a/typescript/utils/tsconfig.json +++ b/typescript/utils/tsconfig.json @@ -4,6 +4,6 @@ "rootDir": "./" }, "exclude": ["./node_modules/", "./dist/", "./tmp.ts"], - "extends": "../../tsconfig.json", + "extends": "../tsconfig.json", "include": ["./index.ts", "./src/*.ts"] }