From 17a6e79ea5da88c0e46c9413d6fd02f5d3feb410 Mon Sep 17 00:00:00 2001 From: Mattie Conover Date: Tue, 29 Aug 2023 16:24:35 -0700 Subject: [PATCH] Rust agent config parsing (#2687) ### Description This does two things: 1) It parses the agent-specific configs for the relayer, validator, and scraper. 2) It removes the rigid structures that had been in place for parsing that were called out earlier and instead parses the serde json values directly. This PR does not cut over to the new versions just yet. That can happen once we start integrating it into the testing and deployment pipelines. ### Drive-by changes - Derive more was added and used where it made sense - The old config macro was removed ### Related issues - Progress on #2215 ### Backward compatibility Yes ### Testing None --- rust/Cargo.lock | 2 + rust/agents/relayer/Cargo.toml | 8 +- .../relayer/src/msg/metadata/ccip_read.rs | 24 +- .../relayer/src/msg/metadata/routing.rs | 16 +- rust/agents/relayer/src/relayer.rs | 30 +- .../relayer/src/settings/matching_list.rs | 27 +- rust/agents/relayer/src/settings/mod.rs | 342 +++++++-- rust/agents/scraper/Cargo.toml | 1 + rust/agents/scraper/src/agent.rs | 10 +- rust/agents/scraper/src/db/mod.rs | 18 +- rust/agents/scraper/src/db/txn.rs | 22 +- rust/agents/scraper/src/settings.rs | 120 +++- rust/agents/validator/Cargo.toml | 1 + rust/agents/validator/src/settings.rs | 237 ++++++- rust/agents/validator/src/validator.rs | 23 +- rust/hyperlane-base/src/agent.rs | 11 +- .../src/settings/deprecated_parser.rs | 151 +++- .../hyperlane-base/src/settings/loader/mod.rs | 60 +- rust/hyperlane-base/src/settings/mod.rs | 123 +--- rust/hyperlane-base/src/settings/parser.rs | 666 ------------------ .../src/settings/parser/json_value_parser.rs | 360 ++++++++++ .../hyperlane-base/src/settings/parser/mod.rs | 389 ++++++++++ rust/hyperlane-base/src/types/s3_storage.rs | 9 +- rust/hyperlane-core/src/config/mod.rs | 8 +- rust/hyperlane-core/src/config/trait_ext.rs | 32 +- typescript/sdk/src/metadata/agentConfig.ts | 74 +- 26 files changed, 1690 insertions(+), 1074 deletions(-) delete mode 100644 rust/hyperlane-base/src/settings/parser.rs create mode 100644 rust/hyperlane-base/src/settings/parser/json_value_parser.rs create mode 100644 rust/hyperlane-base/src/settings/parser/mod.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 60219b4f5e..0d1a65395b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -6623,6 +6623,7 @@ version = "0.1.0" dependencies = [ "async-trait", "config", + "derive_more", "ethers", "eyre", "futures", @@ -9149,6 +9150,7 @@ version = "0.1.0" dependencies = [ "async-trait", "config", + "derive_more", "ethers", "eyre", "futures-util", diff --git a/rust/agents/relayer/Cargo.toml b/rust/agents/relayer/Cargo.toml index a3a277bf9e..488e135af3 100644 --- a/rust/agents/relayer/Cargo.toml +++ b/rust/agents/relayer/Cargo.toml @@ -13,13 +13,17 @@ version.workspace = true async-trait.workspace = true config.workspace = true derive-new.workspace = true +derive_more.workspace = true enum_dispatch.workspace = true ethers-contract.workspace = true ethers.workspace = true eyre.workspace = true futures-util.workspace = true itertools.workspace = true +num-derive.workspace = true +num-traits.workspace = true prometheus.workspace = true +regex.workspace = true reqwest = { workspace = true, features = ["json"] } serde.workspace = true serde_json.workspace = true @@ -28,14 +32,10 @@ thiserror.workspace = true tokio = { workspace = true, features = ["rt", "macros", "parking_lot"] } tracing-futures.workspace = true tracing.workspace = true -regex.workspace = true hyperlane-core = { path = "../../hyperlane-core", features = ["agent"] } hyperlane-base = { path = "../../hyperlane-base" } hyperlane-ethereum = { path = "../../chains/hyperlane-ethereum" } -num-derive.workspace = true -num-traits.workspace = true -derive_more.workspace = true [dev-dependencies] tokio-test.workspace = true diff --git a/rust/agents/relayer/src/msg/metadata/ccip_read.rs b/rust/agents/relayer/src/msg/metadata/ccip_read.rs index e14c0abd1a..7bce48ce40 100644 --- a/rust/agents/relayer/src/msg/metadata/ccip_read.rs +++ b/rust/agents/relayer/src/msg/metadata/ccip_read.rs @@ -1,38 +1,28 @@ use async_trait::async_trait; +use derive_more::Deref; +use derive_new::new; +use ethers::{abi::AbiDecode, core::utils::hex::decode as hex_decode}; +use eyre::Context; +use hyperlane_core::{HyperlaneMessage, RawHyperlaneMessage, H256}; use hyperlane_ethereum::OffchainLookup; +use regex::Regex; use reqwest::Client; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::ops::Deref; - -use derive_new::new; -use eyre::Context; use tracing::{info, instrument}; use super::{BaseMetadataBuilder, MetadataBuilder}; -use ethers::abi::AbiDecode; -use ethers::core::utils::hex::decode as hex_decode; -use hyperlane_core::{HyperlaneMessage, RawHyperlaneMessage, H256}; -use regex::Regex; #[derive(Serialize, Deserialize)] struct OffchainResponse { data: String, } -#[derive(Clone, Debug, new)] +#[derive(Clone, Debug, new, Deref)] pub struct CcipReadIsmMetadataBuilder { base: BaseMetadataBuilder, } -impl Deref for CcipReadIsmMetadataBuilder { - type Target = BaseMetadataBuilder; - - fn deref(&self) -> &Self::Target { - &self.base - } -} - #[async_trait] impl MetadataBuilder for CcipReadIsmMetadataBuilder { #[instrument(err, skip(self))] diff --git a/rust/agents/relayer/src/msg/metadata/routing.rs b/rust/agents/relayer/src/msg/metadata/routing.rs index 8b40130b45..0a55b137f1 100644 --- a/rust/agents/relayer/src/msg/metadata/routing.rs +++ b/rust/agents/relayer/src/msg/metadata/routing.rs @@ -1,27 +1,17 @@ use async_trait::async_trait; -use std::ops::Deref; - +use derive_more::Deref; use derive_new::new; use eyre::Context; -use tracing::instrument; - use hyperlane_core::{HyperlaneMessage, H256}; +use tracing::instrument; use super::{BaseMetadataBuilder, MetadataBuilder}; -#[derive(Clone, Debug, new)] +#[derive(Clone, Debug, new, Deref)] pub struct RoutingIsmMetadataBuilder { base: BaseMetadataBuilder, } -impl Deref for RoutingIsmMetadataBuilder { - type Target = BaseMetadataBuilder; - - fn deref(&self) -> &Self::Target { - &self.base - } -} - #[async_trait] impl MetadataBuilder for RoutingIsmMetadataBuilder { #[instrument(err, skip(self))] diff --git a/rust/agents/relayer/src/relayer.rs b/rust/agents/relayer/src/relayer.rs index fc1dc889a6..37f28c876c 100644 --- a/rust/agents/relayer/src/relayer.rs +++ b/rust/agents/relayer/src/relayer.rs @@ -1,35 +1,33 @@ -use std::fmt::{Debug, Formatter}; use std::{ collections::{HashMap, HashSet}, + fmt::{Debug, Formatter}, sync::Arc, }; use async_trait::async_trait; +use derive_more::AsRef; use eyre::Result; -use hyperlane_base::{MessageContractSync, WatermarkContractSync}; -use tokio::sync::mpsc::UnboundedSender; +use hyperlane_base::{ + db::{HyperlaneRocksDB, DB}, + run_all, BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, MessageContractSync, + WatermarkContractSync, +}; +use hyperlane_core::{HyperlaneDomain, InterchainGasPayment, U256}; use tokio::{ sync::{ - mpsc::{self, UnboundedReceiver}, + mpsc::{self, UnboundedReceiver, UnboundedSender}, RwLock, }, task::JoinHandle, }; use tracing::{info, info_span, instrument::Instrumented, Instrument}; -use hyperlane_base::{ - db::{HyperlaneRocksDB, DB}, - run_all, BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, -}; -use hyperlane_core::{HyperlaneDomain, InterchainGasPayment, U256}; - -use crate::msg::pending_message::MessageSubmissionMetrics; use crate::{ merkle_tree_builder::MerkleTreeBuilder, msg::{ gas_payment::GasPaymentEnforcer, metadata::BaseMetadataBuilder, - pending_message::MessageContext, + pending_message::{MessageContext, MessageSubmissionMetrics}, pending_operation::DynPendingOperation, processor::{MessageProcessor, MessageProcessorMetrics}, serial_submitter::{SerialSubmitter, SerialSubmitterMetrics}, @@ -44,9 +42,11 @@ struct ContextKey { } /// A relayer agent +#[derive(AsRef)] pub struct Relayer { origin_chains: HashSet, destination_chains: HashSet, + #[as_ref] core: HyperlaneAgentCore, message_syncs: HashMap>, interchain_gas_payment_syncs: @@ -79,12 +79,6 @@ impl Debug for Relayer { } } -impl AsRef for Relayer { - fn as_ref(&self) -> &HyperlaneAgentCore { - &self.core - } -} - #[async_trait] #[allow(clippy::unit_arg)] impl BaseAgent for Relayer { diff --git a/rust/agents/relayer/src/settings/matching_list.rs b/rust/agents/relayer/src/settings/matching_list.rs index 1512b102b1..89fcf099c8 100644 --- a/rust/agents/relayer/src/settings/matching_list.rs +++ b/rust/agents/relayer/src/settings/matching_list.rs @@ -1,12 +1,18 @@ -use std::fmt; -use std::fmt::{Debug, Display, Formatter}; -use std::marker::PhantomData; - -use serde::de::{Error, SeqAccess, Visitor}; -use serde::{Deserialize, Deserializer}; - -use hyperlane_core::config::StrOrInt; -use hyperlane_core::{HyperlaneMessage, H160, H256}; +//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape +//! and validations it defines are not applied here, we should mirror them. +//! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK. + +use std::{ + fmt, + fmt::{Debug, Display, Formatter}, + marker::PhantomData, +}; + +use hyperlane_core::{config::StrOrInt, HyperlaneMessage, H160, H256}; +use serde::{ + de::{Error, SeqAccess, Visitor}, + Deserialize, Deserializer, +}; /// Defines a set of patterns for determining if a message should or should not /// be relayed. This is useful for determine if a message matches a given set or @@ -260,9 +266,8 @@ fn parse_addr(addr_str: &str) -> Result { mod test { use hyperlane_core::{H160, H256}; - use crate::settings::matching_list::MatchInfo; - use super::{Filter::*, MatchingList}; + use crate::settings::matching_list::MatchInfo; #[test] fn basic_config() { diff --git a/rust/agents/relayer/src/settings/mod.rs b/rust/agents/relayer/src/settings/mod.rs index 1c36c3d9aa..1d7cf2fd02 100644 --- a/rust/agents/relayer/src/settings/mod.rs +++ b/rust/agents/relayer/src/settings/mod.rs @@ -1,11 +1,25 @@ -//! Configuration +//! Relayer configuration +//! +//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape +//! and validations it defines are not applied here, we should mirror them. +//! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK. use std::{collections::HashSet, path::PathBuf}; +use derive_more::{AsMut, AsRef, Deref, DerefMut}; use eyre::{eyre, Context}; -use hyperlane_base::{decl_settings, settings::Settings}; +use hyperlane_base::{ + impl_loadable_from_settings, + settings::{ + deprecated_parser::DeprecatedRawSettings, + parser::{RawAgentConf, ValueParser}, + Settings, + }, +}; use hyperlane_core::{cfg_unwrap_all, config::*, HyperlaneDomain, U256}; +use itertools::Itertools; use serde::Deserialize; +use serde_json::Value; use tracing::warn; use crate::settings::matching_list::MatchingList; @@ -128,59 +142,78 @@ impl FromRawConf for GasPaymentEnforcementConf { } } -decl_settings!(Relayer, - Parsed { - /// Database path - db: PathBuf, - /// The chain to relay messages from - origin_chains: HashSet, - /// Chains to relay messages to - destination_chains: HashSet, - /// The gas payment enforcement policies - gas_payment_enforcement: Vec, - /// Filter for what messages to relay. - whitelist: MatchingList, - /// Filter for what messages to block. - blacklist: MatchingList, - /// This is optional. If not specified, any amount of gas will be valid, otherwise this - /// is the max allowed gas in wei to relay a transaction. - transaction_gas_limit: Option, - /// List of domain ids to skip transaction gas for. - skip_transaction_gas_limit_for: HashSet, - /// If true, allows local storage based checkpoint syncers. - /// Not intended for production use. - allow_local_checkpoint_syncers: bool, - }, - Raw { - /// Database path (path on the fs) - db: Option, - // Comma separated list of chains to relay between. - relaychains: Option, - // Comma separated list of origin chains. - #[deprecated(note = "Use `relaychains` instead")] - originchainname: Option, - // Comma separated list of destination chains. - #[deprecated(note = "Use `relaychains` instead")] - destinationchainnames: Option, - /// The gas payment enforcement configuration as JSON. Expects an ordered array of `GasPaymentEnforcementConfig`. - gaspaymentenforcement: Option, - /// This is optional. If no whitelist is provided ALL messages will be considered on the - /// whitelist. - whitelist: Option, - /// This is optional. If no blacklist is provided ALL will be considered to not be on - /// the blacklist. - blacklist: Option, - /// This is optional. If not specified, any amount of gas will be valid, otherwise this - /// is the max allowed gas in wei to relay a transaction. - transactiongaslimit: Option, - /// Comma separated List of domain ids to skip transaction gas for. - skiptransactiongaslimitfor: Option, - /// If true, allows local storage based checkpoint syncers. - /// Not intended for production use. Defaults to false. - #[serde(default)] - allowlocalcheckpointsyncers: bool, - } -); +/// Settings for `Relayer` +#[derive(Debug, AsRef, AsMut, Deref, DerefMut)] +pub struct RelayerSettings { + #[as_ref] + #[as_mut] + #[deref] + #[deref_mut] + base: Settings, + + /// Database path + pub db: PathBuf, + /// The chain to relay messages from + pub origin_chains: HashSet, + /// Chains to relay messages to + pub destination_chains: HashSet, + /// The gas payment enforcement policies + pub gas_payment_enforcement: Vec, + /// Filter for what messages to relay. + pub whitelist: MatchingList, + /// Filter for what messages to block. + pub blacklist: MatchingList, + /// This is optional. If not specified, any amount of gas will be valid, otherwise this + /// is the max allowed gas in wei to relay a transaction. + pub transaction_gas_limit: Option, + /// List of domain ids to skip transaction gas for. + pub skip_transaction_gas_limit_for: HashSet, + /// If true, allows local storage based checkpoint syncers. + /// Not intended for production use. + pub allow_local_checkpoint_syncers: bool, +} + +#[derive(Debug, Deserialize, AsMut)] +#[serde(rename_all = "camelCase")] +pub struct DeprecatedRawRelayerSettings { + #[serde(flatten)] + #[as_mut] + base: DeprecatedRawSettings, + /// Database path (path on the fs) + db: Option, + // Comma separated list of chains to relay between. + relaychains: Option, + // Comma separated list of origin chains. + #[deprecated(note = "Use `relaychains` instead")] + originchainname: Option, + // Comma separated list of destination chains. + #[deprecated(note = "Use `relaychains` instead")] + destinationchainnames: Option, + /// The gas payment enforcement configuration as JSON. Expects an ordered array of `GasPaymentEnforcementConfig`. + gaspaymentenforcement: Option, + /// This is optional. If no whitelist is provided ALL messages will be considered on the + /// whitelist. + whitelist: Option, + /// This is optional. If no blacklist is provided ALL will be considered to not be on + /// the blacklist. + blacklist: Option, + /// This is optional. If not specified, any amount of gas will be valid, otherwise this + /// is the max allowed gas in wei to relay a transaction. + transactiongaslimit: Option, + // TODO: this should be a list of chain names to be consistent + /// Comma separated List of domain ids to skip applying the transaction gas limit to. + skiptransactiongaslimitfor: Option, + /// If true, allows local storage based checkpoint syncers. + /// Not intended for production use. Defaults to false. + #[serde(default)] + allowlocalcheckpointsyncers: bool, +} + +impl_loadable_from_settings!(Relayer, DeprecatedRawRelayerSettings -> RelayerSettings); + +#[derive(Debug, Deserialize)] +#[serde(transparent)] +struct RawRelayerSettings(Value); impl FromRawConf for RelayerSettings { fn from_config_filtered( @@ -190,6 +223,205 @@ impl FromRawConf for RelayerSettings { ) -> ConfigResult { let mut err = ConfigParsingError::default(); + let p = ValueParser::new(cwp.clone(), &raw.0); + + let relay_chain_names: Option> = p + .chain(&mut err) + .get_key("relayChains") + .parse_string() + .end() + .map(|v| v.split(',').collect()); + + let base = p + .parse_from_raw_config::>>( + relay_chain_names.as_ref(), + "Parsing base config", + ) + .take_config_err(&mut err); + + let db = p + .chain(&mut err) + .get_opt_key("db") + .parse_from_str("Expected database path") + .unwrap_or_else(|| std::env::current_dir().unwrap().join("hyperlane_db")); + + let (raw_gas_payment_enforcement_path, raw_gas_payment_enforcement) = match p + .get_opt_key("gasPaymentEnforcement") + .take_config_err_flat(&mut err) + { + None => None, + Some(ValueParser { + val: Value::String(policy_str), + cwp, + }) => serde_json::from_str::(policy_str) + .context("Expected JSON string") + .take_err(&mut err, || cwp.clone()) + .map(|v| (cwp, v)), + Some(ValueParser { + val: value @ Value::Array(_), + cwp, + }) => Some((cwp, value.clone())), + Some(_) => Err(eyre!("Expected JSON array or stringified JSON")) + .take_err(&mut err, || cwp.clone()), + } + .unwrap_or_else(|| (&p.cwp + "gas_payment_enforcement", Value::Array(vec![]))); + + let gas_payment_enforcement_parser = ValueParser::new( + raw_gas_payment_enforcement_path, + &raw_gas_payment_enforcement, + ); + let gas_payment_enforcement = gas_payment_enforcement_parser.into_array_iter().map(|itr| { + itr.filter_map(|policy| { + let policy_type = policy.chain(&mut err).get_opt_key("type").parse_string().end(); + let minimum_is_defined = matches!(policy.get_opt_key("minimum"), Ok(Some(_))); + + let matching_list = policy.chain(&mut err).get_opt_key("matchingList").and_then(parse_matching_list).unwrap_or_default(); + + let parse_minimum = |p| GasPaymentEnforcementPolicy::Minimum { payment: p }; + match policy_type { + Some("minimum") => policy.chain(&mut err).get_opt_key("payment").parse_u256().end().map(parse_minimum), + None if minimum_is_defined => policy.chain(&mut err).get_opt_key("payment").parse_u256().end().map(parse_minimum), + Some("none") | None => Some(GasPaymentEnforcementPolicy::None), + Some("onChainFeeQuoting") => { + let gas_fraction = policy.chain(&mut err) + .get_opt_key("gasFraction") + .parse_string() + .map(|v| v.replace(' ', "")) + .unwrap_or_else(|| default_gasfraction().to_owned()); + let (numerator, denominator) = gas_fraction + .split_once('/') + .ok_or_else(|| eyre!("Invalid `gas_fraction` for OnChainFeeQuoting gas payment enforcement policy; expected `numerator / denominator`")) + .take_err(&mut err, || &policy.cwp + "gas_fraction") + .unwrap_or(("1", "1")); + + Some(GasPaymentEnforcementPolicy::OnChainFeeQuoting { + gas_fraction_numerator: numerator + .parse() + .context("Error parsing gas fraction numerator") + .take_err(&mut err, || &policy.cwp + "gas_fraction") + .unwrap_or(1), + gas_fraction_denominator: denominator + .parse() + .context("Error parsing gas fraction denominator") + .take_err(&mut err, || &policy.cwp + "gas_fraction") + .unwrap_or(1), + }) + } + Some(pt) => Err(eyre!("Unknown gas payment enforcement policy type `{pt}`")) + .take_err(&mut err, || cwp + "type"), + }.map(|policy| GasPaymentEnforcementConf { + policy, + matching_list, + }) + }).collect_vec() + }).unwrap_or_default(); + + let whitelist = p + .chain(&mut err) + .get_opt_key("whitelist") + .and_then(parse_matching_list) + .unwrap_or_default(); + let blacklist = p + .chain(&mut err) + .get_opt_key("blacklist") + .and_then(parse_matching_list) + .unwrap_or_default(); + + let transaction_gas_limit = p + .chain(&mut err) + .get_opt_key("transactionGasLimit") + .parse_u256() + .end(); + + let skip_transaction_gas_limit_for_names: HashSet<&str> = p + .chain(&mut err) + .get_opt_key("skipTransactionGasLimitFor") + .parse_string() + .map(|v| v.split(',').collect()) + .unwrap_or_default(); + + let allow_local_checkpoint_syncers = p + .chain(&mut err) + .get_opt_key("allowLocalCheckpointSyncers") + .parse_bool() + .unwrap_or(false); + + cfg_unwrap_all!(cwp, err: [base]); + + let skip_transaction_gas_limit_for = skip_transaction_gas_limit_for_names + .into_iter() + .filter_map(|chain| { + base.lookup_domain(chain) + .context("Missing configuration for a chain in `skipTransactionGasLimitFor`") + .into_config_result(|| cwp + "skip_transaction_gas_limit_for") + .take_config_err(&mut err) + }) + .map(|d| d.id()) + .collect(); + + let relay_chains: HashSet = relay_chain_names + .unwrap_or_default() + .into_iter() + .filter_map(|chain| { + base.lookup_domain(chain) + .context("Missing configuration for a chain in `relayChains`") + .into_config_result(|| cwp + "relayChains") + .take_config_err(&mut err) + }) + .collect(); + + err.into_result(RelayerSettings { + base, + db, + origin_chains: relay_chains.clone(), + destination_chains: relay_chains, + gas_payment_enforcement, + whitelist, + blacklist, + transaction_gas_limit, + skip_transaction_gas_limit_for, + allow_local_checkpoint_syncers, + }) + } +} + +fn parse_matching_list(p: ValueParser) -> ConfigResult { + let mut err = ConfigParsingError::default(); + + let raw_list = match &p { + ValueParser { + val: Value::String(matching_list_str), + cwp, + } => serde_json::from_str::(matching_list_str) + .context("Expected JSON string") + .take_err(&mut err, || cwp.clone()), + ValueParser { + val: value @ Value::Array(_), + .. + } => Some((*value).clone()), + _ => Err(eyre!("Expected JSON array or stringified JSON")) + .take_err(&mut err, || p.cwp.clone()), + }; + let Some(raw_list) = raw_list else { + return err.into_result(MatchingList::default()); + }; + let p = ValueParser::new(p.cwp.clone(), &raw_list); + let ml = p + .parse_value::("Expected matching list") + .take_config_err(&mut err) + .unwrap_or_default(); + + err.into_result(ml) +} + +impl FromRawConf for RelayerSettings { + fn from_config_filtered( + raw: DeprecatedRawRelayerSettings, + cwp: &ConfigPath, + _filter: (), + ) -> ConfigResult { + let mut err = ConfigParsingError::default(); + let gas_payment_enforcement = raw .gaspaymentenforcement .and_then(|j| { diff --git a/rust/agents/scraper/Cargo.toml b/rust/agents/scraper/Cargo.toml index 0c5386ade0..0daeea7e50 100644 --- a/rust/agents/scraper/Cargo.toml +++ b/rust/agents/scraper/Cargo.toml @@ -12,6 +12,7 @@ version.workspace = true [dependencies] async-trait.workspace = true config.workspace = true +derive_more.workspace = true ethers.workspace = true eyre.workspace = true futures.workspace = true diff --git a/rust/agents/scraper/src/agent.rs b/rust/agents/scraper/src/agent.rs index dbff2cbadd..b582f8e2ef 100644 --- a/rust/agents/scraper/src/agent.rs +++ b/rust/agents/scraper/src/agent.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; +use derive_more::AsRef; use hyperlane_base::{ run_all, settings::IndexSettings, BaseAgent, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, @@ -12,9 +13,10 @@ use tracing::{info_span, instrument::Instrumented, trace, Instrument}; use crate::{chain_scraper::HyperlaneSqlDb, db::ScraperDb, settings::ScraperSettings}; /// A message explorer scraper agent -#[derive(Debug)] +#[derive(Debug, AsRef)] #[allow(unused)] pub struct Scraper { + #[as_ref] core: HyperlaneAgentCore, contract_sync_metrics: Arc, metrics: Arc, @@ -133,12 +135,6 @@ impl Scraper { } } -impl AsRef for Scraper { - fn as_ref(&self) -> &HyperlaneAgentCore { - &self.core - } -} - /// Create a function to spawn task that syncs contract events macro_rules! spawn_sync_task { ($name:ident, $cursor: ident, $label:literal) => { diff --git a/rust/agents/scraper/src/db/mod.rs b/rust/agents/scraper/src/db/mod.rs index b18cc5e50e..41a99e349d 100644 --- a/rust/agents/scraper/src/db/mod.rs +++ b/rust/agents/scraper/src/db/mod.rs @@ -1,14 +1,10 @@ -use std::ops::Deref; - -use eyre::Result; -use sea_orm::{Database, DbConn}; -use tracing::instrument; - pub use block::*; pub use block_cursor::BlockCursor; -use hyperlane_core::TxnInfo; +use eyre::Result; pub use message::*; pub use payment::*; +use sea_orm::{Database, DbConn}; +use tracing::instrument; pub use txn::*; #[allow(clippy::all)] @@ -21,14 +17,6 @@ mod message; mod payment; mod txn; -impl Deref for StorableTxn { - type Target = TxnInfo; - - fn deref(&self) -> &Self::Target { - &self.info - } -} - /// Database interface to the message explorer database for the scraper. This is /// focused on writing data to the database. #[derive(Clone, Debug)] diff --git a/rust/agents/scraper/src/db/txn.rs b/rust/agents/scraper/src/db/txn.rs index 2c6cc63ea7..ff0e70f2b5 100644 --- a/rust/agents/scraper/src/db/txn.rs +++ b/rust/agents/scraper/src/db/txn.rs @@ -1,20 +1,24 @@ use std::collections::HashMap; +use derive_more::Deref; use eyre::{eyre, Context, Result}; -use sea_orm::sea_query::OnConflict; -use sea_orm::{prelude::*, ActiveValue::*, DeriveColumn, EnumIter, Insert, NotSet, QuerySelect}; -use tracing::{debug, instrument, trace}; - use hyperlane_core::{TxnInfo, H256}; - -use crate::conversions::{address_to_bytes, h256_to_bytes, u256_to_decimal}; -use crate::date_time; -use crate::db::ScraperDb; +use sea_orm::{ + prelude::*, sea_query::OnConflict, ActiveValue::*, DeriveColumn, EnumIter, Insert, NotSet, + QuerySelect, +}; +use tracing::{debug, instrument, trace}; use super::generated::transaction; +use crate::{ + conversions::{address_to_bytes, h256_to_bytes, u256_to_decimal}, + date_time, + db::ScraperDb, +}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Deref)] pub struct StorableTxn { + #[deref] pub info: TxnInfo, pub block_id: i64, } diff --git a/rust/agents/scraper/src/settings.rs b/rust/agents/scraper/src/settings.rs index 75d9a0369a..360c8f1fe7 100644 --- a/rust/agents/scraper/src/settings.rs +++ b/rust/agents/scraper/src/settings.rs @@ -1,20 +1,57 @@ +//! Scraper configuration. +//! +//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape +//! and validations it defines are not applied here, we should mirror them. +//! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK. + +use std::{collections::HashSet, default::Default}; + +use derive_more::{AsMut, AsRef, Deref, DerefMut}; use eyre::{eyre, Context}; -use hyperlane_base::{decl_settings, settings::Settings}; -use hyperlane_core::{config::*, HyperlaneDomain}; +use hyperlane_base::{ + impl_loadable_from_settings, + settings::{ + deprecated_parser::DeprecatedRawSettings, + parser::{RawAgentConf, ValueParser}, + Settings, + }, +}; +use hyperlane_core::{cfg_unwrap_all, config::*, HyperlaneDomain}; use itertools::Itertools; +use serde::Deserialize; +use serde_json::Value; -decl_settings!(Scraper, - Parsed { - db: String, - chains_to_scrape: Vec, - }, - Raw { - /// Database connection string - db: Option, - /// Comma separated list of chains to scrape - chainstoscrape: Option, - } -); +/// Settings for `Scraper` +#[derive(Debug, AsRef, AsMut, Deref, DerefMut)] +pub struct ScraperSettings { + #[as_ref] + #[as_mut] + #[deref] + #[deref_mut] + base: Settings, + + pub db: String, + pub chains_to_scrape: Vec, +} + +/// Raw settings for `Scraper` +#[derive(Debug, Deserialize, AsMut)] +#[serde(rename_all = "camelCase")] +pub struct DeprecatedRawScraperSettings { + #[serde(flatten, default)] + #[as_mut] + base: DeprecatedRawSettings, + /// Database connection string + db: Option, + /// Comma separated list of chains to scrape + chainstoscrape: Option, +} + +impl_loadable_from_settings!(Scraper, DeprecatedRawScraperSettings -> ScraperSettings); + +#[derive(Debug, Deserialize)] +#[serde(transparent)] +struct RawScraperSettings(Value); impl FromRawConf for ScraperSettings { fn from_config_filtered( @@ -24,6 +61,61 @@ impl FromRawConf for ScraperSettings { ) -> ConfigResult { let mut err = ConfigParsingError::default(); + let p = ValueParser::new(cwp.clone(), &raw.0); + + let chains_names_to_scrape: Option> = p + .chain(&mut err) + .get_key("chainsToScrape") + .parse_string() + .end() + .map(|s| s.split(',').collect()); + + let base = p + .parse_from_raw_config::>>( + chains_names_to_scrape.as_ref(), + "Parsing base config", + ) + .take_config_err(&mut err); + + let db = p + .chain(&mut err) + .get_key("db") + .parse_string() + .end() + .map(|v| v.to_owned()); + + let chains_to_scrape = if let (Some(base), Some(chains)) = (&base, chains_names_to_scrape) { + chains + .into_iter() + .filter_map(|chain| { + base.lookup_domain(chain) + .context("Missing configuration for a chain in `chainsToScrape`") + .into_config_result(|| cwp + "chains_to_scrape") + .take_config_err(&mut err) + }) + .collect() + } else { + Default::default() + }; + + cfg_unwrap_all!(&p.cwp, err: [base, db]); + + err.into_result(Self { + base, + db, + chains_to_scrape, + }) + } +} + +impl FromRawConf for ScraperSettings { + fn from_config_filtered( + raw: DeprecatedRawScraperSettings, + cwp: &ConfigPath, + _filter: (), + ) -> ConfigResult { + let mut err = ConfigParsingError::default(); + let db = raw .db .ok_or_else(|| eyre!("Missing `db` connection string")) diff --git a/rust/agents/validator/Cargo.toml b/rust/agents/validator/Cargo.toml index 0bec3cc7b4..4607a1a9b3 100644 --- a/rust/agents/validator/Cargo.toml +++ b/rust/agents/validator/Cargo.toml @@ -12,6 +12,7 @@ version.workspace = true [dependencies] async-trait.workspace = true config.workspace = true +derive_more.workspace = true ethers.workspace = true eyre.workspace = true futures-util.workspace = true diff --git a/rust/agents/validator/src/settings.rs b/rust/agents/validator/src/settings.rs index a475d37286..ea5b1f7893 100644 --- a/rust/agents/validator/src/settings.rs +++ b/rust/agents/validator/src/settings.rs @@ -1,48 +1,77 @@ -//! Configuration +//! Validator configuration. +//! +//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape +//! and validations it defines are not applied here, we should mirror them. +//! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK. -use std::{path::PathBuf, time::Duration}; +use std::{collections::HashSet, path::PathBuf, time::Duration}; +use derive_more::{AsMut, AsRef, Deref, DerefMut}; use eyre::{eyre, Context}; use hyperlane_base::{ - decl_settings, + impl_loadable_from_settings, settings::{ - parser::{RawCheckpointSyncerConf, RawSignerConf}, + deprecated_parser::{ + DeprecatedRawCheckpointSyncerConf, DeprecatedRawSettings, DeprecatedRawSignerConf, + }, + parser::{RawAgentConf, RawAgentSignerConf, ValueParser}, CheckpointSyncerConf, Settings, SignerConf, }, }; use hyperlane_core::{cfg_unwrap_all, config::*, HyperlaneDomain, HyperlaneDomainProtocol}; +use serde::Deserialize; +use serde_json::Value; -decl_settings!(Validator, - Parsed { - /// Database path - db: PathBuf, - /// Chain to validate messages on - origin_chain: HyperlaneDomain, - /// The validator attestation signer - validator: SignerConf, - /// The checkpoint syncer configuration - checkpoint_syncer: CheckpointSyncerConf, - /// The reorg_period in blocks - reorg_period: u64, - /// How frequently to check for new checkpoints - interval: Duration, - }, - Raw { - /// Database path (path on the fs) - db: Option, - // Name of the chain to validate message on - originchainname: Option, - /// The validator attestation signer - #[serde(default)] - validator: RawSignerConf, - /// The checkpoint syncer configuration - checkpointsyncer: Option, - /// The reorg_period in blocks - reorgperiod: Option, - /// How frequently to check for new checkpoints - interval: Option, - }, -); +/// Settings for `Validator` +#[derive(Debug, AsRef, AsMut, Deref, DerefMut)] +pub struct ValidatorSettings { + #[as_ref] + #[as_mut] + #[deref] + #[deref_mut] + base: Settings, + + /// Database path + pub db: PathBuf, + /// Chain to validate messages on + pub origin_chain: HyperlaneDomain, + /// The validator attestation signer + pub validator: SignerConf, + /// The checkpoint syncer configuration + pub checkpoint_syncer: CheckpointSyncerConf, + /// The reorg_period in blocks + pub reorg_period: u64, + /// How frequently to check for new checkpoints + pub interval: Duration, +} + +/// Raw settings for `Validator` +#[derive(Debug, Deserialize, AsMut)] +#[serde(rename_all = "camelCase")] +pub struct DeprecatedRawValidatorSettings { + #[serde(flatten, default)] + #[as_mut] + base: DeprecatedRawSettings, + /// Database path (path on the fs) + db: Option, + // Name of the chain to validate message on + originchainname: Option, + /// The validator attestation signer + #[serde(default)] + validator: DeprecatedRawSignerConf, + /// The checkpoint syncer configuration + checkpointsyncer: Option, + /// The reorg_period in blocks + reorgperiod: Option, + /// How frequently to check for new checkpoints + interval: Option, +} + +impl_loadable_from_settings!(Validator, DeprecatedRawValidatorSettings -> ValidatorSettings); + +#[derive(Debug, Deserialize)] +#[serde(transparent)] +struct RawValidatorSettings(Value); impl FromRawConf for ValidatorSettings { fn from_config_filtered( @@ -52,6 +81,144 @@ impl FromRawConf for ValidatorSettings { ) -> ConfigResult { let mut err = ConfigParsingError::default(); + let p = ValueParser::new(cwp.clone(), &raw.0); + + let origin_chain_name = p + .chain(&mut err) + .get_key("originChainName") + .parse_string() + .end(); + + let origin_chain_name_set = origin_chain_name.map(|s| HashSet::from([s])); + let base = p + .parse_from_raw_config::>>( + origin_chain_name_set.as_ref(), + "Expected valid base agent configuration", + ) + .take_config_err(&mut err); + + let origin_chain = if let (Some(base), Some(origin_chain_name)) = (&base, origin_chain_name) + { + base.lookup_domain(origin_chain_name) + .context("Missing configuration for the origin chain") + .take_err(&mut err, || cwp + "origin_chain_name") + } else { + None + }; + + let validator = p + .chain(&mut err) + .get_key("validator") + .parse_from_raw_config::( + (), + "Expected valid validator configuration", + ) + .end(); + + let db = p + .chain(&mut err) + .get_opt_key("db") + .parse_from_str("Expected db file path") + .unwrap_or_else(|| { + std::env::current_dir() + .unwrap() + .join(format!("validator_db_{}", origin_chain_name.unwrap_or(""))) + }); + + let checkpoint_syncer = p + .chain(&mut err) + .get_key("checkpointSyncer") + .and_then(parse_checkpoint_syncer) + .end(); + + let interval = p + .chain(&mut err) + .get_opt_key("interval") + .parse_u64() + .map(Duration::from_secs) + .unwrap_or(Duration::from_secs(5)); + + cfg_unwrap_all!(cwp, err: [origin_chain_name]); + + let reorg_period = p + .chain(&mut err) + .get_key("chains") + .get_key(origin_chain_name) + .get_opt_key("blocks") + .get_opt_key("reorgPeriod") + .parse_u64() + .unwrap_or(1); + + cfg_unwrap_all!(cwp, err: [base, origin_chain, validator, checkpoint_syncer]); + + err.into_result(Self { + base, + db, + origin_chain, + validator, + checkpoint_syncer, + reorg_period, + interval, + }) + } +} + +/// Expects ValidatorAgentConfig.checkpointSyncer +fn parse_checkpoint_syncer(syncer: ValueParser) -> ConfigResult { + let mut err = ConfigParsingError::default(); + let syncer_type = syncer.chain(&mut err).get_key("type").parse_string().end(); + + match syncer_type { + Some("localStorage") => { + let path = syncer + .chain(&mut err) + .get_key("path") + .parse_from_str("Expected checkpoint syncer file path") + .end(); + cfg_unwrap_all!(&syncer.cwp, err: [path]); + err.into_result(CheckpointSyncerConf::LocalStorage { path }) + } + Some("s3") => { + let bucket = syncer + .chain(&mut err) + .get_key("bucket") + .parse_string() + .end() + .map(str::to_owned); + let region = syncer + .chain(&mut err) + .get_key("region") + .parse_from_str("Expected aws region") + .end(); + let folder = syncer + .chain(&mut err) + .get_opt_key("folder") + .parse_string() + .end() + .map(str::to_owned); + + cfg_unwrap_all!(&syncer.cwp, err: [bucket, region]); + err.into_result(CheckpointSyncerConf::S3 { + bucket, + region, + folder, + }) + } + Some(_) => { + Err(eyre!("Unknown checkpoint syncer type")).into_config_result(|| &syncer.cwp + "type") + } + None => Err(err), + } +} + +impl FromRawConf for ValidatorSettings { + fn from_config_filtered( + raw: DeprecatedRawValidatorSettings, + cwp: &ConfigPath, + _filter: (), + ) -> ConfigResult { + let mut err = ConfigParsingError::default(); + let validator = raw .validator .parse_config::(&cwp.join("validator")) diff --git a/rust/agents/validator/src/validator.rs b/rust/agents/validator/src/validator.rs index 9b4cad8ee1..4180cc3b77 100644 --- a/rust/agents/validator/src/validator.rs +++ b/rust/agents/validator/src/validator.rs @@ -1,12 +1,8 @@ -use std::num::NonZeroU64; -use std::sync::Arc; -use std::time::Duration; +use std::{num::NonZeroU64, sync::Arc, time::Duration}; use async_trait::async_trait; +use derive_more::AsRef; use eyre::Result; -use tokio::{task::JoinHandle, time::sleep}; -use tracing::{error, info, info_span, instrument::Instrumented, warn, Instrument}; - use hyperlane_base::{ db::{HyperlaneRocksDB, DB}, run_all, BaseAgent, CheckpointSyncer, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore, @@ -18,15 +14,19 @@ use hyperlane_core::{ ValidatorAnnounce, H256, U256, }; use hyperlane_ethereum::{SingletonSigner, SingletonSignerHandle}; +use tokio::{task::JoinHandle, time::sleep}; +use tracing::{error, info, info_span, instrument::Instrumented, warn, Instrument}; use crate::{ - settings::ValidatorSettings, submit::ValidatorSubmitter, submit::ValidatorSubmitterMetrics, + settings::ValidatorSettings, + submit::{ValidatorSubmitter, ValidatorSubmitterMetrics}, }; /// A validator agent -#[derive(Debug)] +#[derive(Debug, AsRef)] pub struct Validator { origin_chain: HyperlaneDomain, + #[as_ref] core: HyperlaneAgentCore, db: HyperlaneRocksDB, message_sync: Arc, @@ -39,13 +39,6 @@ pub struct Validator { interval: Duration, checkpoint_syncer: Arc, } - -impl AsRef for Validator { - fn as_ref(&self) -> &HyperlaneAgentCore { - &self.core - } -} - #[async_trait] impl BaseAgent for Validator { const AGENT_NAME: &'static str = "validator"; diff --git a/rust/hyperlane-base/src/agent.rs b/rust/hyperlane-base/src/agent.rs index 1cbeb51e4f..540a32254c 100644 --- a/rust/hyperlane-base/src/agent.rs +++ b/rust/hyperlane-base/src/agent.rs @@ -3,6 +3,7 @@ use std::{env, fmt::Debug, sync::Arc}; use async_trait::async_trait; use eyre::{Report, Result}; use futures_util::future::select_all; +use hyperlane_core::config::*; use tokio::task::JoinHandle; use tracing::{debug_span, instrument::Instrumented, Instrument}; @@ -17,11 +18,11 @@ pub struct HyperlaneAgentCore { pub settings: Settings, } -/// Settings of an agent. -pub trait NewFromSettings: AsRef + Sized { +/// Settings of an agent defined from configuration +pub trait LoadableFromSettings: AsRef + Sized { /// Create a new instance of these settings by reading the configs and env /// vars. - fn new() -> hyperlane_core::config::ConfigResult; + fn load() -> ConfigResult; } /// A fundamental agent which does not make any assumptions about the tools @@ -32,7 +33,7 @@ pub trait BaseAgent: Send + Sync + Debug { const AGENT_NAME: &'static str; /// The settings object for this agent - type Settings: NewFromSettings; + type Settings: LoadableFromSettings; /// Instantiate the agent from the standard settings object async fn from_settings(settings: Self::Settings, metrics: Arc) -> Result @@ -62,7 +63,7 @@ pub async fn agent_main() -> Result<()> { color_eyre::install()?; } - let settings = A::Settings::new()?; + let settings = A::Settings::load()?; let core_settings: &Settings = settings.as_ref(); let metrics = settings.as_ref().metrics(A::AGENT_NAME)?; diff --git a/rust/hyperlane-base/src/settings/deprecated_parser.rs b/rust/hyperlane-base/src/settings/deprecated_parser.rs index c9f2d8624e..4282f72d74 100644 --- a/rust/hyperlane-base/src/settings/deprecated_parser.rs +++ b/rust/hyperlane-base/src/settings/deprecated_parser.rs @@ -2,7 +2,10 @@ // TODO: Remove this module once we have finished migrating to the new format. -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, +}; use ethers_prometheus::middleware::PrometheusMiddlewareConf; use eyre::{eyre, Context}; @@ -11,8 +14,8 @@ use serde::Deserialize; use super::envs::*; use crate::settings::{ - chains::IndexSettings, parser::RawSignerConf, trace::TracingConfig, ChainConf, - ChainConnectionConf, CoreContractAddresses, Settings, SignerConf, + chains::IndexSettings, trace::TracingConfig, ChainConf, ChainConnectionConf, + CheckpointSyncerConf, CoreContractAddresses, Settings, SignerConf, }; /// Raw base settings. @@ -20,7 +23,7 @@ use crate::settings::{ #[serde(rename_all = "camelCase")] pub struct DeprecatedRawSettings { chains: Option>, - defaultsigner: Option, + defaultsigner: Option, metrics: Option, tracing: Option, } @@ -203,7 +206,7 @@ impl FromRawConf for IndexSettings { pub struct DeprecatedRawChainConf { name: Option, domain: Option, - pub(super) signer: Option, + pub(super) signer: Option, finality_blocks: Option, addresses: Option, #[serde(flatten, default)] @@ -292,3 +295,141 @@ impl FromRawConf for ChainConf { }) } } + +/// Raw signer types +#[derive(Debug, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct DeprecatedRawSignerConf { + #[serde(rename = "type")] + signer_type: Option, + key: Option, + id: Option, + region: Option, +} + +/// Raw checkpoint syncer types +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum DeprecatedRawCheckpointSyncerConf { + /// A local checkpoint syncer + LocalStorage { + /// Path + path: Option, + }, + /// A checkpoint syncer on S3 + S3 { + /// Bucket name + bucket: Option, + /// S3 Region + region: Option, + /// Folder name inside bucket - defaults to the root of the bucket + folder: Option, + }, + /// Unknown checkpoint syncer type was specified + #[serde(other)] + Unknown, +} + +impl FromRawConf for SignerConf { + fn from_config_filtered( + raw: DeprecatedRawSignerConf, + cwp: &ConfigPath, + _filter: (), + ) -> ConfigResult { + let key_path = || cwp + "key"; + let region_path = || cwp + "region"; + + match raw.signer_type.as_deref() { + Some("hexKey") => Ok(Self::HexKey { + key: raw + .key + .ok_or_else(|| eyre!("Missing `key` for HexKey signer")) + .into_config_result(key_path)? + .parse() + .into_config_result(key_path)?, + }), + Some("aws") => Ok(Self::Aws { + id: raw + .id + .ok_or_else(|| eyre!("Missing `id` for Aws signer")) + .into_config_result(|| cwp + "id")?, + region: raw + .region + .ok_or_else(|| eyre!("Missing `region` for Aws signer")) + .into_config_result(region_path)? + .parse() + .into_config_result(region_path)?, + }), + Some(t) => Err(eyre!("Unknown signer type `{t}`")).into_config_result(|| cwp + "type"), + None if raw.key.is_some() => Ok(Self::HexKey { + key: raw.key.unwrap().parse().into_config_result(key_path)?, + }), + None if raw.id.is_some() | raw.region.is_some() => Ok(Self::Aws { + id: raw + .id + .ok_or_else(|| eyre!("Missing `id` for Aws signer")) + .into_config_result(|| cwp + "id")?, + region: raw + .region + .ok_or_else(|| eyre!("Missing `region` for Aws signer")) + .into_config_result(region_path)? + .parse() + .into_config_result(region_path)?, + }), + None => Ok(Self::Node), + } + } +} + +impl FromRawConf for CheckpointSyncerConf { + fn from_config_filtered( + raw: DeprecatedRawCheckpointSyncerConf, + cwp: &ConfigPath, + _filter: (), + ) -> ConfigResult { + match raw { + DeprecatedRawCheckpointSyncerConf::LocalStorage { path } => { + let path: PathBuf = path + .ok_or_else(|| eyre!("Missing `path` for LocalStorage checkpoint syncer")) + .into_config_result(|| cwp + "path")? + .parse() + .into_config_result(|| cwp + "path")?; + if !path.exists() { + std::fs::create_dir_all(&path) + .with_context(|| { + format!( + "Failed to create local checkpoint syncer storage directory at {:?}", + path + ) + }) + .into_config_result(|| cwp + "path")?; + } else if !path.is_dir() { + Err(eyre!( + "LocalStorage checkpoint syncer path is not a directory" + )) + .into_config_result(|| cwp + "path")?; + } + Ok(Self::LocalStorage { path }) + } + DeprecatedRawCheckpointSyncerConf::S3 { + bucket, + folder, + region, + } => Ok(Self::S3 { + bucket: bucket + .ok_or_else(|| eyre!("Missing `bucket` for S3 checkpoint syncer")) + .into_config_result(|| cwp + "bucket")?, + folder, + region: region + .ok_or_else(|| eyre!("Missing `region` for S3 checkpoint syncer")) + .into_config_result(|| cwp + "region")? + .parse() + .into_config_result(|| cwp + "region")?, + }), + DeprecatedRawCheckpointSyncerConf::Unknown => { + Err(eyre!("Missing `type` for checkpoint syncer")) + .into_config_result(|| cwp + "type") + } + } + } +} diff --git a/rust/hyperlane-base/src/settings/loader/mod.rs b/rust/hyperlane-base/src/settings/loader/mod.rs index 389630819d..fe6ee9f34e 100644 --- a/rust/hyperlane-base/src/settings/loader/mod.rs +++ b/rust/hyperlane-base/src/settings/loader/mod.rs @@ -1,26 +1,37 @@ -use std::{collections::HashMap, env, error::Error, path::PathBuf}; +//! Load a settings object from the config locations. + +use std::{collections::HashMap, env, error::Error, fmt::Debug, path::PathBuf}; use config::{Config, Environment as DeprecatedEnvironment, File}; use convert_case::{Case, Casing}; use eyre::{bail, Context, Result}; +use hyperlane_core::config::*; use itertools::Itertools; -use serde::Deserialize; +use serde::de::DeserializeOwned; -use super::deprecated_parser::DeprecatedRawSettings; use crate::settings::loader::deprecated_arguments::DeprecatedCommandLineArguments; mod arguments; mod deprecated_arguments; mod environment; +/// Deserialize a settings object from the configs. +pub fn load_settings(name: &str) -> ConfigResult +where + T: DeserializeOwned + Debug, + R: FromRawConf, +{ + let root_path = ConfigPath::default(); + let raw = + load_settings_object::(name, &[]).into_config_result(|| root_path.clone())?; + raw.parse_config(&root_path) +} + /// Load a settings object from the config locations. /// Further documentation can be found in the `settings` module. -pub(crate) fn load_settings_object<'de, T, S>( - agent_prefix: &str, - ignore_prefixes: &[S], -) -> Result +fn load_settings_object(agent_prefix: &str, ignore_prefixes: &[S]) -> Result where - T: Deserialize<'de> + AsMut, + T: DeserializeOwned, S: AsRef, { // Derive additional prefix from agent name @@ -103,28 +114,21 @@ where } }; - match Config::try_deserialize::(config_deserializer) { - Ok(mut cfg) => { - cfg.as_mut(); - Ok(cfg) - } - Err(err) => { - let mut err = if let Some(source_err) = err.source() { - let source = format!("Config error source: {source_err}"); - Err(err).context(source) - } else { - Err(err.into()) - }; - - for cfg_path in base_config_sources.iter().chain(config_file_paths.iter()) { - err = err.with_context(|| format!("Config loaded: {cfg_path}")); - } - - println!("Error during deserialization, showing the config for debugging: {formatted_config}"); + Config::try_deserialize::(config_deserializer).or_else(|err| { + let mut err = if let Some(source_err) = err.source() { + let source = format!("Config error source: {source_err}"); + Err(err).context(source) + } else { + Err(err.into()) + }; - err.context("Config deserialization error, please check the config reference (https://docs.hyperlane.xyz/docs/operators/agent-configuration/configuration-reference)") + for cfg_path in base_config_sources.iter().chain(config_file_paths.iter()) { + err = err.with_context(|| format!("Config loaded: {cfg_path}")); } - } + + println!("Error during deserialization, showing the config for debugging: {formatted_config}"); + err.context("Config deserialization error, please check the config reference (https://docs.hyperlane.xyz/docs/operators/agent-configuration/configuration-reference)") + }) } /// Load a settings object from the config locations and re-join the components with the standard diff --git a/rust/hyperlane-base/src/settings/mod.rs b/rust/hyperlane-base/src/settings/mod.rs index f542b87e12..70a617b362 100644 --- a/rust/hyperlane-base/src/settings/mod.rs +++ b/rust/hyperlane-base/src/settings/mod.rs @@ -76,16 +76,12 @@ //! 5. Arguments passed to the agent on the command line. //! E.g. `--originChainName ethereum` -use std::fmt::Debug; - pub use base::*; pub use chains::*; pub use checkpoint_syncer::*; -use hyperlane_core::config::*; /// Export this so they don't need to import paste. #[doc(hidden)] pub use paste; -use serde::Deserialize; pub use signers::*; pub use trace::*; @@ -100,7 +96,7 @@ pub(crate) mod aws_credentials; mod base; /// Chain configuration mod chains; -pub(crate) mod loader; +pub mod loader; /// Signer configuration mod signers; /// Tracing subscriber management @@ -110,116 +106,21 @@ mod checkpoint_syncer; pub mod deprecated_parser; pub mod parser; -#[macro_export] -/// Declare a new settings block -/// -/// This macro declares a settings struct for an agent. The new settings block -/// contains a [`crate::Settings`] and any other specified attributes. -/// -/// Please note that integers must be specified as `String` in order to allow -/// them to be configured via env var. They must then be parsed in the -/// [`Agent::from_settings`](crate::agent::Agent::from_settings) -/// method. -/// -/// ### Usage +/// Declare that an agent can be constructed from settings. /// +/// E.g. /// ```ignore -/// decl_settings!(Validator { -/// validator: SignerConf, -/// checkpointsyncer: CheckpointSyncerConf, -/// reorgperiod: String, -/// interval: String, -/// }); +/// impl_loadable_from_settings!(MyAgent, RawSettingsForMyAgent -> SettingsForMyAgent); /// ``` -macro_rules! decl_settings { - ( - $name:ident, - Parsed { - $($(#[$parsed_tags:meta])* $parsed_prop:ident: $parsed_type:ty,)* - }, - Raw { - $($(#[$raw_tags:meta])* $raw_prop:ident: $raw_type:ty,)* - }$(,)? - ) => { - hyperlane_base::settings::paste::paste! { - #[doc = "Settings for `" $name "`"] - #[derive(Debug)] - pub struct [<$name Settings>] { - base: hyperlane_base::settings::Settings, - $( - $(#[$parsed_tags])* - pub(crate) $parsed_prop: $parsed_type, - )* - } - - #[doc = "Raw settings for `" $name "`"] - #[derive(Debug, serde::Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct [] { - #[serde(flatten, default)] - base: hyperlane_base::settings::deprecated_parser::DeprecatedRawSettings, - $( - $(#[$raw_tags])* - $raw_prop: $raw_type, - )* - } - - impl AsMut for [] { - fn as_mut(&mut self) -> &mut hyperlane_base::settings::deprecated_parser::DeprecatedRawSettings { - &mut self.base - } - } - - // ensure the settings struct implements `FromRawConf` - const _: fn() = || { - fn assert_impl() - where - T: ?Sized + hyperlane_core::config::FromRawConf<[]> - {} - - assert_impl::<[<$name Settings>]>(); - }; - - impl std::ops::Deref for [<$name Settings>] { - type Target = hyperlane_base::settings::Settings; - - fn deref(&self) -> &Self::Target { - &self.base - } - } - - impl AsRef for [<$name Settings>] { - fn as_ref(&self) -> &hyperlane_base::settings::Settings { - &self.base - } - } - - impl AsMut for [<$name Settings>] { - fn as_mut(&mut self) -> &mut hyperlane_base::settings::Settings { - &mut self.base - } - } - - impl hyperlane_base::NewFromSettings<> for [<$name Settings>] { - /// See `load_settings_object` for more information about how settings are loaded. - fn new() -> hyperlane_core::config::ConfigResult { - hyperlane_base::settings::_new_settings::<[], [<$name Settings>]>(stringify!($name)) - } +#[macro_export] +macro_rules! impl_loadable_from_settings { + ($agent:ident, $settingsparser:ident -> $settingsobj:ident) => { + impl hyperlane_base::LoadableFromSettings for $settingsobj { + fn load() -> hyperlane_core::config::ConfigResult { + hyperlane_base::settings::loader::load_settings::<$settingsparser, Self>( + stringify!($agent), + ) } } }; } - -/// Static logic called by the decl_settings! macro. Do not call directly! -#[doc(hidden)] -pub fn _new_settings<'de, T, R>(name: &str) -> ConfigResult -where - T: Deserialize<'de> + AsMut + Debug, - R: FromRawConf, -{ - use crate::settings::loader::load_settings_object; - let root_path = ConfigPath::default(); - let raw = - load_settings_object::(name, &[]).into_config_result(|| root_path.clone())?; - raw.parse_config(&root_path) -} diff --git a/rust/hyperlane-base/src/settings/parser.rs b/rust/hyperlane-base/src/settings/parser.rs deleted file mode 100644 index d1aa162909..0000000000 --- a/rust/hyperlane-base/src/settings/parser.rs +++ /dev/null @@ -1,666 +0,0 @@ -//! This module is responsible for parsing the agent's settings. -//! -//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape -//! and validations it defines are not applied here, we should mirror them. -//! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK. - -#![allow(dead_code)] // TODO(2214): remove before PR merge - -use std::{ - cmp::Reverse, - collections::{HashMap, HashSet}, - path::PathBuf, -}; - -use eyre::{eyre, Context}; -use hyperlane_core::{ - cfg_unwrap_all, config::*, utils::hex_or_base58_to_h256, HyperlaneDomain, - HyperlaneDomainProtocol, IndexMode, -}; -use itertools::Itertools; -use serde::Deserialize; -use serde_json::json; - -pub use super::envs::*; -use crate::settings::{ - chains::IndexSettings, trace::TracingConfig, ChainConf, ChainConnectionConf, - CheckpointSyncerConf, CoreContractAddresses, Settings, SignerConf, -}; - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawAgentConf { - metrics_port: StrOrInt, - chains: HashMap, - default_signer: RawSignerConf, - default_rpc_consensus_type: Option, - #[serde(default)] - log: RawAgentLogConf, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawAgentChainMetadataConf { - // -- AgentChainMetadata -- - #[serde(default)] - custom_rpc_urls: HashMap, - rpc_consensus_type: Option, - signer: Option, - #[serde(default)] - index: RawAgentChainMetadataIndexConf, - - // -- ChainMetadata -- - protocol: Option, - chain_id: Option, - domain_id: Option, - name: Option, - display_name: Option, - display_name_short: Option, - logo_uri: Option, - #[serde(default)] - native_token: RawNativeTokenConf, - #[serde(default)] - rpc_urls: Vec, - #[serde(default)] - block_explorers: Vec, - #[serde(default)] - blocks: RawBlockConf, - #[serde(default)] - transaction_overrides: HashMap, - gas_currency_coin_geco_id: Option, - gnosis_safe_transaction_service_url: Option, - #[serde(default)] - is_testnet: bool, - - // -- HyperlaneDeploymentArtifacts -- - mailbox: Option, - interchain_gas_paymaster: Option, - validator_announce: Option, - interchain_security_module: Option, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawAgentChainMetadataIndexConf { - from: Option, - chunk: Option, - mode: Option, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawNativeTokenConf { - name: Option, - symbol: Option, - decimals: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawRpcUrlConf { - http: Option, - ws: Option, - #[serde(default)] - pagination: RawPaginationConf, - #[serde(default)] - retry: RawRetryConfig, - - // -- AgentChainMetadata Ext for `custom_rpc_urls` -- - priority: Option, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawPaginationConf { - max_block_range: Option, - min_block_number: Option, - max_block_age: Option, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawRetryConfig { - max_requests: Option, - base_retry_ms: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawBlockExplorerConf { - name: Option, - url: Option, - api_url: Option, - api_key: Option, - family: Option, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawBlockConf { - confirmations: Option, - reorg_period: Option, - estimate_block_time: Option, -} - -/// Raw signer types -#[derive(Debug, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -pub struct RawSignerConf { - #[serde(rename = "type")] - signer_type: Option, - key: Option, - id: Option, - region: Option, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct RawAgentLogConf { - format: Option, - level: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -enum RawRpcConsensusType { - Fallback, - Quorum, - #[serde(other)] - Unknown, -} - -/// Raw checkpoint syncer types -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum RawCheckpointSyncerConf { - /// A local checkpoint syncer - LocalStorage { - /// Path - path: Option, - }, - /// A checkpoint syncer on S3 - S3 { - /// Bucket name - bucket: Option, - /// Folder name inside bucket - defaults to the root of the bucket - folder: Option, - /// S3 Region - region: Option, - }, - /// Unknown checkpoint syncer type was specified - #[serde(other)] - Unknown, -} - -impl FromRawConf>> for Settings { - fn from_config_filtered( - raw: RawAgentConf, - cwp: &ConfigPath, - filter: Option<&HashSet<&str>>, - ) -> Result { - let mut err = ConfigParsingError::default(); - - let metrics_port = raw - .metrics_port - .try_into() - .take_err(&mut err, || cwp + "metrics_port") - .unwrap_or(9090); - - let tracing = raw - .log - .parse_config(&cwp.join("log")) - .take_config_err(&mut err); - - let raw_chains = if let Some(filter) = filter { - raw.chains - .into_iter() - .filter(|(k, _)| filter.contains(&**k)) - .collect() - } else { - raw.chains - }; - - let chains_path = cwp + "chains"; - let chains = raw_chains - .into_iter() - .filter_map(|(name, chain)| { - let cwp = &chains_path + &name; - chain - .parse_config::(&cwp) - .take_config_err(&mut err) - .and_then(|c| { - (c.domain.name() == name) - .then_some((name, c)) - .ok_or_else(|| { - eyre!("detected chain name mismatch, the config may be corrupted") - }) - .take_err(&mut err, || &cwp + "name") - }) - }) - .collect(); - - cfg_unwrap_all!(cwp, err: [tracing]); - - err.into_result(Self { - chains, - metrics_port, - tracing, - }) - } -} - -impl FromRawConf for TracingConfig { - fn from_config_filtered( - raw: RawAgentLogConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let fmt = raw - .format - .and_then(|fmt| serde_json::from_value(json!(fmt)).take_err(&mut err, || cwp + "fmt")) - .unwrap_or_default(); - - let level = raw - .level - .and_then(|lvl| serde_json::from_value(json!(lvl)).take_err(&mut err, || cwp + "level")) - .unwrap_or_default(); - - err.into_result(Self { fmt, level }) - } -} - -impl FromRawConf for ChainConf { - fn from_config_filtered( - raw: RawAgentChainMetadataConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let domain = (&raw).parse_config(cwp).take_config_err(&mut err); - let addresses = (&raw).parse_config(cwp).take_config_err(&mut err); - - let signer = raw.signer.and_then(|s| { - s.parse_config(&cwp.join("signer")) - .take_config_err(&mut err) - }); - - // TODO(2214): is it correct to define finality blocks as `confirmations` and not `reorgPeriod`? - // TODO(2214): should we rename `finalityBlocks` in ChainConf? - let finality_blocks = raw - .blocks - .confirmations - .ok_or_else(|| eyre!("Missing `confirmations`")) - .take_err(&mut err, || cwp + "confirmations") - .and_then(|v| { - v.try_into() - .context("Invalid `confirmations`, expected integer") - .take_err(&mut err, || cwp + "confirmations") - }); - - let index: Option = raw - .index - .parse_config_with_filter(&cwp.join("index"), domain.as_ref()) - .take_config_err(&mut err); - - let rpcs: Vec<(ConfigPath, RawRpcUrlConf)> = if raw.custom_rpc_urls.is_empty() { - let cwp = cwp + "rpc_urls"; - // if no custom rpc urls are set, use the default rpc urls - raw.rpc_urls - .into_iter() - .enumerate() - .map(|(i, v)| (&cwp + i.to_string(), v)) - .collect() - } else { - // use the custom defined urls, sorted by highest prio first - let cwp = cwp + "custom_rpc_urls"; - raw.custom_rpc_urls - .into_iter() - .map(|(k, v)| { - ( - v.priority - .as_ref() - .and_then(|v| v.try_into().take_err(&mut err, || &cwp + &k)) - .unwrap_or(0i32), - k, - v, - ) - }) - .sorted_unstable_by_key(|(p, _, _)| Reverse(*p)) - .map(|(_, k, v)| (&cwp + k, v)) - .collect() - }; - - if rpcs.is_empty() { - err.push( - cwp + "rpc_urls", - eyre!("Missing base rpc definitions for chain"), - ); - err.push( - cwp + "custom_rpc_urls", - eyre!("Also missing rpc overrides for chain"), - ); - } - - cfg_unwrap_all!(cwp, err: [index, finality_blocks, domain]); - - let connection: Option = match domain.domain_protocol() { - HyperlaneDomainProtocol::Ethereum => { - if rpcs.len() <= 1 { - rpcs.into_iter() - .next() - .and_then(|(cwp, rpc)| rpc.http.map(|url| (cwp, url))) - .and_then(|(cwp, url)| url.parse().take_err(&mut err, || cwp)) - .map(|url| { - ChainConnectionConf::Ethereum(h_eth::ConnectionConf::Http { url }) - }) - } else { - let urls = rpcs - .into_iter() - .filter_map(|(cwp, rpc)| { - let cwp = || &cwp + "http"; - rpc.http - .ok_or_else(|| { - eyre!( - "missing http url for multi-rpc configured ethereum client" - ) - }) - .take_err(&mut err, cwp) - .and_then(|url| url.parse().take_err(&mut err, cwp)) - }) - .collect_vec(); - - match raw - .rpc_consensus_type { - Some(RawRpcConsensusType::Fallback) => { - Some(h_eth::ConnectionConf::HttpFallback { urls }) - } - Some(RawRpcConsensusType::Quorum) => { - Some(h_eth::ConnectionConf::HttpQuorum { urls }) - } - Some(RawRpcConsensusType::Unknown) => { - err.push(cwp + "rpc_consensus_type", eyre!("unknown rpc consensus type")); - None - } - None => { - err.push(cwp + "rpc_consensus_type", eyre!("missing consensus type for multi-rpc configured ethereum client")); - None - }, - } - .map(ChainConnectionConf::Ethereum) - } - } - HyperlaneDomainProtocol::Fuel => rpcs - .into_iter() - .next() - .and_then(|(cwp, rpc)| rpc.http.map(|url| (cwp, url))) - .and_then(|(cwp, url)| url.parse().take_err(&mut err, || cwp)) - .map(|url| ChainConnectionConf::Fuel(h_fuel::ConnectionConf { url })), - HyperlaneDomainProtocol::Sealevel => rpcs - .into_iter() - .next() - .and_then(|(cwp, rpc)| rpc.http.map(|url| (cwp, url))) - .and_then(|(cwp, url)| url.parse().take_err(&mut err, || cwp)) - .map(|url| ChainConnectionConf::Sealevel(h_sealevel::ConnectionConf { url })), - }; - - cfg_unwrap_all!(cwp, err: [addresses, connection]); - err.into_result(Self { - domain, - signer, - finality_blocks, - addresses, - connection, - metrics_conf: Default::default(), - index, - }) - } -} - -impl FromRawConf<&RawAgentChainMetadataConf> for HyperlaneDomain { - fn from_config_filtered( - raw: &RawAgentChainMetadataConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let chain_id = raw - .chain_id - .as_ref() - .ok_or_else(|| eyre!("Missing `chainId`")) - .take_err(&mut err, || cwp + "chain_id") - .and_then(|d| { - d.try_into() - .context("Invalid `chainId`, expected integer") - .take_err(&mut err, || cwp + "chain_id") - }); - - let domain_id = raw - .domain_id - .as_ref() - .and_then(|d| { - d.try_into() - .context("Invalid `domainId`, expected integer") - .take_err(&mut err, || cwp + "domain_id") - }) - // default to chain id if domain id is not set - .or(chain_id); - - let protocol = raw - .protocol - .as_deref() - .ok_or_else(|| eyre!("Missing `protocol`")) - .take_err(&mut err, || cwp + "protocol") - .and_then(|d| { - HyperlaneDomainProtocol::try_from(d) - .context("Invalid (or unknown) `protocol`") - .take_err(&mut err, || cwp + "protocol") - }); - - let name = raw - .name - .as_deref() - .ok_or_else(|| eyre!("Missing chain `name`")) - .take_err(&mut err, || cwp + "name"); - - cfg_unwrap_all!(cwp, err: [domain_id, protocol, name]); - - let domain = Self::from_config(domain_id, name, protocol) - .context("Invalid domain data") - .take_err(&mut err, || cwp.clone()); - - cfg_unwrap_all!(cwp, err: [domain]); - err.into_result(domain) - } -} - -impl FromRawConf<&RawAgentChainMetadataConf> for CoreContractAddresses { - fn from_config_filtered( - raw: &RawAgentChainMetadataConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let mailbox = raw - .mailbox - .as_ref() - .ok_or_else(|| eyre!("Missing `mailbox` address")) - .take_err(&mut err, || cwp + "mailbox") - .and_then(|v| hex_or_base58_to_h256(v).take_err(&mut err, || cwp + "mailbox")); - - let interchain_gas_paymaster = raw - .interchain_gas_paymaster - .as_ref() - .ok_or_else(|| eyre!("Missing `interchainGasPaymaster` address")) - .take_err(&mut err, || cwp + "interchain_gas_paymaster") - .and_then(|v| { - hex_or_base58_to_h256(v).take_err(&mut err, || cwp + "interchain_gas_paymaster") - }); - - let validator_announce = raw - .validator_announce - .as_ref() - .ok_or_else(|| eyre!("Missing `validatorAnnounce` address")) - .take_err(&mut err, || cwp + "validator_announce") - .and_then(|v| { - hex_or_base58_to_h256(v).take_err(&mut err, || cwp + "validator_announce") - }); - - cfg_unwrap_all!(cwp, err: [mailbox, interchain_gas_paymaster, validator_announce]); - err.into_result(Self { - mailbox, - interchain_gas_paymaster, - validator_announce, - }) - } -} - -impl FromRawConf> for IndexSettings { - fn from_config_filtered( - raw: RawAgentChainMetadataIndexConf, - cwp: &ConfigPath, - domain: Option<&HyperlaneDomain>, - ) -> ConfigResult { - let mut err = ConfigParsingError::default(); - - let from = raw - .from - .and_then(|v| v.try_into().take_err(&mut err, || cwp + "from")) - .unwrap_or_default(); - - let chunk_size = raw - .chunk - .and_then(|v| v.try_into().take_err(&mut err, || cwp + "chunk")) - .unwrap_or(1999); - - let mode = raw - .mode - .map(serde_json::Value::from) - .and_then(|m| { - serde_json::from_value(m) - .context("Invalid mode") - .take_err(&mut err, || cwp + "mode") - }) - .or_else(|| { - // attempt to choose a reasonable default - domain.and_then(|d| match d.domain_protocol() { - HyperlaneDomainProtocol::Ethereum => Some(IndexMode::Block), - HyperlaneDomainProtocol::Sealevel => Some(IndexMode::Sequence), - _ => None, - }) - }) - .unwrap_or_default(); - - err.into_result(Self { - from, - chunk_size, - mode, - }) - } -} - -impl FromRawConf for SignerConf { - fn from_config_filtered( - raw: RawSignerConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - let key_path = || cwp + "key"; - let region_path = || cwp + "region"; - - match raw.signer_type.as_deref() { - Some("hexKey") => Ok(Self::HexKey { - key: raw - .key - .ok_or_else(|| eyre!("Missing `key` for HexKey signer")) - .into_config_result(key_path)? - .parse() - .into_config_result(key_path)?, - }), - Some("aws") => Ok(Self::Aws { - id: raw - .id - .ok_or_else(|| eyre!("Missing `id` for Aws signer")) - .into_config_result(|| cwp + "id")?, - region: raw - .region - .ok_or_else(|| eyre!("Missing `region` for Aws signer")) - .into_config_result(region_path)? - .parse() - .into_config_result(region_path)?, - }), - Some(t) => Err(eyre!("Unknown signer type `{t}`")).into_config_result(|| cwp + "type"), - None if raw.key.is_some() => Ok(Self::HexKey { - key: raw.key.unwrap().parse().into_config_result(key_path)?, - }), - None if raw.id.is_some() | raw.region.is_some() => Ok(Self::Aws { - id: raw - .id - .ok_or_else(|| eyre!("Missing `id` for Aws signer")) - .into_config_result(|| cwp + "id")?, - region: raw - .region - .ok_or_else(|| eyre!("Missing `region` for Aws signer")) - .into_config_result(region_path)? - .parse() - .into_config_result(region_path)?, - }), - None => Ok(Self::Node), - } - } -} - -impl FromRawConf for CheckpointSyncerConf { - fn from_config_filtered( - raw: RawCheckpointSyncerConf, - cwp: &ConfigPath, - _filter: (), - ) -> ConfigResult { - match raw { - RawCheckpointSyncerConf::LocalStorage { path } => { - let path: PathBuf = path - .ok_or_else(|| eyre!("Missing `path` for LocalStorage checkpoint syncer")) - .into_config_result(|| cwp + "path")? - .parse() - .into_config_result(|| cwp + "path")?; - if !path.exists() { - std::fs::create_dir_all(&path) - .with_context(|| { - format!( - "Failed to create local checkpoint syncer storage directory at {:?}", - path - ) - }) - .into_config_result(|| cwp + "path")?; - } else if !path.is_dir() { - Err(eyre!( - "LocalStorage checkpoint syncer path is not a directory" - )) - .into_config_result(|| cwp + "path")?; - } - Ok(Self::LocalStorage { path }) - } - RawCheckpointSyncerConf::S3 { - bucket, - folder, - region, - } => Ok(Self::S3 { - bucket: bucket - .ok_or_else(|| eyre!("Missing `bucket` for S3 checkpoint syncer")) - .into_config_result(|| cwp + "bucket")?, - folder, - region: region - .ok_or_else(|| eyre!("Missing `region` for S3 checkpoint syncer")) - .into_config_result(|| cwp + "region")? - .parse() - .into_config_result(|| cwp + "region")?, - }), - RawCheckpointSyncerConf::Unknown => Err(eyre!("Missing `type` for checkpoint syncer")) - .into_config_result(|| cwp + "type"), - } - } -} diff --git a/rust/hyperlane-base/src/settings/parser/json_value_parser.rs b/rust/hyperlane-base/src/settings/parser/json_value_parser.rs new file mode 100644 index 0000000000..cefc8ccaee --- /dev/null +++ b/rust/hyperlane-base/src/settings/parser/json_value_parser.rs @@ -0,0 +1,360 @@ +use std::{fmt::Debug, str::FromStr}; + +use convert_case::{Case, Casing}; +use derive_new::new; +use eyre::{eyre, Context}; +use hyperlane_core::{config::*, utils::hex_or_base58_to_h256, H256, U256}; +use serde::de::{DeserializeOwned, StdError}; +use serde_json::Value; + +pub use super::super::envs::*; + +/// A serde-json value config parsing utility. +#[derive(Debug, Clone, new)] +pub struct ValueParser<'v> { + /// Path to the current value from the root. + pub cwp: ConfigPath, + /// Reference to the serde JSON value. + pub val: &'v Value, +} + +impl<'v> ValueParser<'v> { + /// Create a new value parser chain. + pub fn chain<'e>(&self, err: &'e mut ConfigParsingError) -> ParseChain<'e, ValueParser<'v>> { + ParseChain(Some(self.clone()), err) + } + + /// Get a value at the given key and verify that it is present. + pub fn get_key(&self, key: &str) -> ConfigResult> { + self.get_opt_key(key)? + .ok_or_else(|| eyre!("Expected key `{key}` to be defined")) + .into_config_result(|| &self.cwp + key.to_case(Case::Snake)) + } + + /// Get a value at the given key allowing for it to not be set. + pub fn get_opt_key(&self, key: &str) -> ConfigResult>> { + let cwp = &self.cwp + key.to_case(Case::Snake); + match self.val { + Value::Object(obj) => Ok(obj.get(key).map(|val| Self { + val, + cwp: cwp.clone(), + })), + _ => Err(eyre!("Expected an object type")), + } + .into_config_result(|| cwp) + } + + /// Create an iterator over all (key, value) tuples. + pub fn into_obj_iter( + self, + ) -> ConfigResult)> + 'v> { + let cwp = self.cwp.clone(); + match self.val { + Value::Object(obj) => Ok(obj.iter().map(move |(k, v)| { + ( + k.clone(), + Self { + val: v, + cwp: &cwp + k.to_case(Case::Snake), + }, + ) + })), + _ => Err(eyre!("Expected an object type")), + } + .into_config_result(|| self.cwp) + } + + /// Create an iterator over all array elements. + pub fn into_array_iter(self) -> ConfigResult>> { + let cwp = self.cwp.clone(); + match self.val { + Value::Array(arr) => Ok(arr.iter().enumerate().map(move |(i, v)| Self { + val: v, + cwp: &cwp + i.to_string(), + })), + _ => Err(eyre!("Expected an array type")), + } + .into_config_result(|| self.cwp) + } + + /// Parse a u64 value allowing for it to be represented as string or number. + pub fn parse_u64(&self) -> ConfigResult { + match self.val { + Value::Number(num) => num + .as_u64() + .ok_or_else(|| eyre!("Excepted an unsigned integer, got number `{num}`")), + Value::String(s) => s + .parse() + .with_context(|| format!("Expected an unsigned integer, got string `{s}`")), + _ => Err(eyre!("Expected an unsigned integer, got `{:?}`", self.val)), + } + .into_config_result(|| self.cwp.clone()) + } + + /// Parse an i64 value allowing for it to be represented as string or number. + pub fn parse_i64(&self) -> ConfigResult { + match self.val { + Value::Number(num) => num + .as_i64() + .ok_or_else(|| eyre!("Excepted a signed integer, got number `{num}`")), + Value::String(s) => s + .parse() + .with_context(|| format!("Expected a signed integer, got string `{s}`")), + _ => Err(eyre!("Expected an signed integer, got `{:?}`", self.val)), + } + .into_config_result(|| self.cwp.clone()) + } + + /// Parse an f64 value allowing for it to be represented as string or number and verifying it is + /// not nan or infinite. + pub fn parse_f64(&self) -> ConfigResult { + let num = self.parse_f64_unchecked()?; + if num.is_nan() { + Err(eyre!("Expected a floating point number, got NaN")) + } else if num.is_infinite() { + Err(eyre!("Expected a floating point number, got Infinity")) + } else { + Ok(num) + } + .into_config_result(|| self.cwp.clone()) + } + + /// Parse an i64 value allowing for it to be represented as string or number. + pub fn parse_f64_unchecked(&self) -> ConfigResult { + match self.val { + Value::Number(num) => num + .as_f64() + .ok_or_else(|| eyre!("Excepted a floating point number, got number `{num}`")), + Value::String(s) => s + .parse() + .with_context(|| format!("Expected a floating point number, got string `{s}`")), + _ => Err(eyre!( + "Expected floating point number, got `{:?}`", + self.val + )), + } + .into_config_result(|| self.cwp.clone()) + } + + /// Parse a u32 value allowing for it to be represented as string or number. + pub fn parse_u32(&self) -> ConfigResult { + self.parse_u64()? + .try_into() + .context("Expected a 32-bit unsigned integer") + .into_config_result(|| self.cwp.clone()) + } + + /// Parse a u16 value allowing for it to be represented as string or number. + pub fn parse_u16(&self) -> ConfigResult { + self.parse_u64()? + .try_into() + .context("Expected a 16-bit unsigned integer") + .into_config_result(|| self.cwp.clone()) + } + + /// Parse an i32 value allowing for it to be represented as string or number. + pub fn parse_i32(&self) -> ConfigResult { + self.parse_i64()? + .try_into() + .context("Expected a 32-bit signed integer") + .into_config_result(|| self.cwp.clone()) + } + + /// Parse a u256 value allowing for it to be represented as string or number. + pub fn parse_u256(&self) -> ConfigResult { + match self.val { + Value::String(s) => s.parse().context("Expected a valid U256 string"), + Value::Number(n) => { + if let Some(n) = n.as_u64() { + Ok(n.into()) + } else { + Err(eyre!("Expected an unsigned integer")) + } + } + _ => Err(eyre!("Expected a U256, got `{:?}`", self.val)), + } + .into_config_result(|| self.cwp.clone()) + } + + /// Parse a boolean value allowing for it to be represented as string or bool. + pub fn parse_bool(&self) -> ConfigResult { + match self.val { + Value::Bool(b) => Ok(*b), + Value::String(s) => match s.to_ascii_lowercase().as_str() { + "true" => Ok(true), + "false" => Ok(false), + s => Err(eyre!("Expected a boolean, got string `{s}`")), + }, + _ => Err(eyre!("Expected a boolean, got `{:?}`", self.val)), + } + .into_config_result(|| self.cwp.clone()) + } + + /// Parse a string value. + pub fn parse_string(&self) -> ConfigResult<&'v str> { + match self.val { + Value::String(s) => Ok(s.as_str()), + _ => Err(eyre!("Expected a string, got `{:?}`", self.val)), + } + .into_config_result(|| self.cwp.clone()) + } + + /// Parse an address hash allowing for it to be represented as a hex or base58 string. + pub fn parse_address_hash(&self) -> ConfigResult { + match self.val { + Value::String(s) => { + hex_or_base58_to_h256(s).context("Expected a valid address hash in hex or base58") + } + _ => Err(eyre!("Expected an address string, got `{:?}`", self.val)), + } + .into_config_result(|| self.cwp.clone()) + } + + /// Parse a private key allowing for it to be represented as a hex or base58 string. + pub fn parse_private_key(&self) -> ConfigResult { + match self.val { + Value::String(s) => { + hex_or_base58_to_h256(s).context("Expected a valid private key in hex or base58") + } + _ => Err(eyre!("Expected a private key string")), + } + .into_config_result(|| self.cwp.clone()) + } + + /// Use serde to parse a value. + pub fn parse_value(&self, ctx: &'static str) -> ConfigResult { + serde_json::from_value(self.val.clone()) + .context(ctx) + .into_config_result(|| self.cwp.clone()) + } + + /// Use `FromStr`/`str::parse` to parse a value. + pub fn parse_from_str(&self, ctx: &'static str) -> ConfigResult + where + T: FromStr, + T::Err: StdError + Send + Sync + 'static, + { + self.parse_string()? + .parse() + .context(ctx) + .into_config_result(|| self.cwp.clone()) + } + + /// Use FromRawConf to parse a value. + pub fn parse_from_raw_config(&self, filter: F, ctx: &'static str) -> ConfigResult + where + O: FromRawConf, + T: Debug + DeserializeOwned, + F: Default, + { + O::from_config_filtered(self.parse_value::(ctx)?, &self.cwp, filter) + } +} + +pub struct ParseChain<'e, T>(Option, &'e mut ConfigParsingError); +macro_rules! define_basic_parse { + ($($name:ident: $ty:ty),+) => { + impl<'v, 'e> ParseChain<'e, ValueParser<'v>> { + $(pub fn $name(self) -> ParseChain<'e, $ty> { + self.and_then(|v| v.$name()) + })* + } + } +} + +define_basic_parse!( + parse_u64: u64, + parse_i64: i64, + parse_f64: f64, + parse_f64_unchecked: f64, + parse_u32: u32, + parse_u16: u16, + parse_i32: i32, + parse_u256: U256, + parse_bool: bool, + parse_string: &'v str, + parse_address_hash: H256, + parse_private_key: H256 +); + +impl<'v, 'e> ParseChain<'e, ValueParser<'v>> { + pub fn get_key(self, key: &str) -> Self { + self.and_then(|v| v.get_key(key)) + } + + pub fn get_opt_key(self, key: &str) -> Self { + Self( + self.0 + .and_then(|v| v.get_opt_key(key).take_config_err(self.1)) + .flatten(), + self.1, + ) + } + + pub fn parse_value(self, ctx: &'static str) -> ParseChain<'e, T> { + self.and_then(|v| v.parse_value::(ctx)) + } + + pub fn into_obj_iter(self) -> Option)> + 'v> { + self.and_then(|v| v.into_obj_iter()).end() + } + + pub fn into_array_iter(self) -> Option>> { + self.and_then(|v| v.into_array_iter()).end() + } + + pub fn parse_from_str(self, ctx: &'static str) -> ParseChain<'e, T> + where + T: FromStr, + T::Err: StdError + Send + Sync + 'static, + { + ParseChain( + self.0 + .and_then(|v| v.parse_from_str::(ctx).take_config_err(self.1)), + self.1, + ) + } + + pub fn parse_from_raw_config(self, filter: F, ctx: &'static str) -> ParseChain<'e, O> + where + O: FromRawConf, + T: Debug + DeserializeOwned, + F: Default, + { + self.and_then(|v| v.parse_from_raw_config::(filter, ctx)) + } +} + +impl<'e, T> ParseChain<'e, T> { + pub fn from_option(val: Option, err: &'e mut ConfigParsingError) -> Self { + Self(val, err) + } + + pub fn and_then(self, f: impl FnOnce(T) -> ConfigResult) -> ParseChain<'e, O> { + ParseChain(self.0.and_then(|v| f(v).take_config_err(self.1)), self.1) + } + + pub fn map(self, f: impl FnOnce(T) -> O) -> ParseChain<'e, O> { + ParseChain(self.0.map(f), self.1) + } + + pub fn end(self) -> Option { + self.0 + } + + pub fn unwrap_or(self, default: T) -> T { + self.0.unwrap_or(default) + } + + pub fn unwrap_or_else(self, f: impl FnOnce() -> T) -> T { + self.0.unwrap_or_else(f) + } +} + +impl<'e, T: Default> ParseChain<'e, T> { + pub fn unwrap_or_default(self) -> T { + self.0.unwrap_or_default() + } +} + +// pub struct ParseChainIter<'e, T, I: Iterator>(Option, &'e mut ConfigParsingError); diff --git a/rust/hyperlane-base/src/settings/parser/mod.rs b/rust/hyperlane-base/src/settings/parser/mod.rs new file mode 100644 index 0000000000..3153701513 --- /dev/null +++ b/rust/hyperlane-base/src/settings/parser/mod.rs @@ -0,0 +1,389 @@ +//! This module is responsible for parsing the agent's settings. +//! +//! The correct settings shape is defined in the TypeScript SDK metadata. While the the exact shape +//! and validations it defines are not applied here, we should mirror them. +//! ANY CHANGES HERE NEED TO BE REFLECTED IN THE TYPESCRIPT SDK. + +#![allow(dead_code)] // TODO(2214): remove before PR merge + +use std::{ + cmp::Reverse, + collections::{HashMap, HashSet}, + default::Default, +}; + +use eyre::{eyre, Context}; +use hyperlane_core::{ + cfg_unwrap_all, config::*, HyperlaneDomain, HyperlaneDomainProtocol, IndexMode, +}; +use itertools::Itertools; +use serde::Deserialize; +use serde_json::Value; + +pub use self::json_value_parser::ValueParser; +pub use super::envs::*; +use crate::settings::{ + chains::IndexSettings, parser::json_value_parser::ParseChain, trace::TracingConfig, ChainConf, + ChainConnectionConf, CoreContractAddresses, Settings, SignerConf, +}; + +mod json_value_parser; + +/// The base agent config +#[derive(Debug, Deserialize)] +#[serde(transparent)] +pub struct RawAgentConf(Value); + +impl FromRawConf>> for Settings { + fn from_config_filtered( + raw: RawAgentConf, + cwp: &ConfigPath, + filter: Option<&HashSet<&str>>, + ) -> Result { + let mut err = ConfigParsingError::default(); + + let p = ValueParser::new(cwp.clone(), &raw.0); + + let metrics_port = p + .chain(&mut err) + .get_opt_key("metricsPort") + .parse_u16() + .unwrap_or(9090); + + let fmt = p + .chain(&mut err) + .get_opt_key("log") + .get_opt_key("format") + .parse_value("Invalid log format") + .unwrap_or_default(); + + let level = p + .chain(&mut err) + .get_opt_key("log") + .get_opt_key("level") + .parse_value("Invalid log level") + .unwrap_or_default(); + + let raw_chains: Vec<(String, ValueParser)> = if let Some(filter) = filter { + p.chain(&mut err) + .get_opt_key("chains") + .into_obj_iter() + .map(|v| v.filter(|(k, _)| filter.contains(&**k)).collect()) + } else { + p.chain(&mut err) + .get_opt_key("chains") + .into_obj_iter() + .map(|v| v.collect()) + } + .unwrap_or_default(); + + let default_signer = p + .chain(&mut err) + .get_opt_key("defaultSigner") + .and_then(parse_signer) + .end(); + + let chains: HashMap = raw_chains + .into_iter() + .filter_map(|(name, chain)| { + parse_chain(chain, &name) + .take_config_err(&mut err) + .map(|v| (name, v)) + }) + .map(|(name, mut chain)| { + if let Some(default_signer) = &default_signer { + chain.signer.get_or_insert_with(|| default_signer.clone()); + } + (name, chain) + }) + .collect(); + + err.into_result(Self { + chains, + metrics_port, + tracing: TracingConfig { fmt, level }, + }) + } +} + +/// The chain name and ChainMetadata +fn parse_chain(chain: ValueParser, name: &str) -> ConfigResult { + let mut err = ConfigParsingError::default(); + + let domain = parse_domain(chain.clone(), name).take_config_err(&mut err); + let signer = chain + .chain(&mut err) + .get_opt_key("signer") + .and_then(parse_signer) + .end(); + + // TODO(2214): is it correct to define finality blocks as `confirmations` and not `reorgPeriod`? + // TODO(2214): should we rename `finalityBlocks` in ChainConf? + let finality_blocks = chain + .chain(&mut err) + .get_opt_key("blocks") + .get_key("confirmations") + .parse_u32() + .unwrap_or(1); + + let rpcs: Vec = + if let Some(custom_rpc_urls) = chain.get_opt_key("customRpcUrls").unwrap_or_default() { + // use the custom defined urls, sorted by highest prio first + custom_rpc_urls.chain(&mut err).into_obj_iter().map(|itr| { + itr.map(|(_, url)| { + ( + url.chain(&mut err) + .get_opt_key("priority") + .parse_i32() + .unwrap_or(0), + url, + ) + }) + .sorted_unstable_by_key(|(p, _)| Reverse(*p)) + .map(|(_, url)| url) + .collect() + }) + } else { + // if no custom rpc urls are set, use the default rpc urls + chain + .chain(&mut err) + .get_key("rpcUrls") + .into_array_iter() + .map(Iterator::collect) + } + .unwrap_or_default(); + + if rpcs.is_empty() { + err.push( + &chain.cwp + "rpc_urls", + eyre!("Missing base rpc definitions for chain"), + ); + err.push( + &chain.cwp + "custom_rpc_urls", + eyre!("Also missing rpc overrides for chain"), + ); + } + + let from = chain + .chain(&mut err) + .get_opt_key("index") + .get_opt_key("from") + .parse_u32() + .unwrap_or(0); + let chunk_size = chain + .chain(&mut err) + .get_opt_key("index") + .get_opt_key("chunk") + .parse_u32() + .unwrap_or(1999); + let mode = chain + .chain(&mut err) + .get_opt_key("index") + .get_opt_key("mode") + .parse_value("Invalid index mode") + .unwrap_or_else(|| { + domain + .as_ref() + .and_then(|d| match d.domain_protocol() { + HyperlaneDomainProtocol::Ethereum => Some(IndexMode::Block), + HyperlaneDomainProtocol::Sealevel => Some(IndexMode::Sequence), + _ => None, + }) + .unwrap_or_default() + }); + + let mailbox = chain + .chain(&mut err) + .get_key("mailbox") + .parse_address_hash() + .end(); + let interchain_gas_paymaster = chain + .chain(&mut err) + .get_key("interchainGasPaymaster") + .parse_address_hash() + .end(); + let validator_announce = chain + .chain(&mut err) + .get_key("validatorAnnounce") + .parse_address_hash() + .end(); + + cfg_unwrap_all!(&chain.cwp, err: [domain]); + + let connection: Option = match domain.domain_protocol() { + HyperlaneDomainProtocol::Ethereum => { + if rpcs.len() <= 1 { + let into_connection = + |url| ChainConnectionConf::Ethereum(h_eth::ConnectionConf::Http { url }); + rpcs.into_iter().next().and_then(|rpc| { + rpc.chain(&mut err) + .get_key("http") + .parse_from_str("Invalid http url") + .end() + .map(into_connection) + }) + } else { + let urls = rpcs + .into_iter() + .filter_map(|rpc| { + rpc.chain(&mut err) + .get_key("http") + .parse_from_str("Invalid http url") + .end() + }) + .collect_vec(); + + let rpc_consensus_type = chain + .chain(&mut err) + .get_opt_key("rpcConsensusType") + .parse_string() + .unwrap_or("fallback"); + match rpc_consensus_type { + "fallback" => Some(h_eth::ConnectionConf::HttpFallback { urls }), + "quorum" => Some(h_eth::ConnectionConf::HttpQuorum { urls }), + ty => Err(eyre!("unknown rpc consensus type `{ty}`")) + .take_err(&mut err, || &chain.cwp + "rpc_consensus_type"), + } + .map(ChainConnectionConf::Ethereum) + } + } + HyperlaneDomainProtocol::Fuel => ParseChain::from_option(rpcs.into_iter().next(), &mut err) + .get_key("http") + .parse_from_str("Invalid http url") + .end() + .map(|url| ChainConnectionConf::Fuel(h_fuel::ConnectionConf { url })), + HyperlaneDomainProtocol::Sealevel => { + ParseChain::from_option(rpcs.into_iter().next(), &mut err) + .get_key("http") + .parse_from_str("Invalod http url") + .end() + .map(|url| ChainConnectionConf::Sealevel(h_sealevel::ConnectionConf { url })) + } + }; + + cfg_unwrap_all!(&chain.cwp, err: [connection, mailbox, interchain_gas_paymaster, validator_announce]); + err.into_result(ChainConf { + domain, + signer, + finality_blocks, + addresses: CoreContractAddresses { + mailbox, + interchain_gas_paymaster, + validator_announce, + }, + connection, + metrics_conf: Default::default(), + index: IndexSettings { + from, + chunk_size, + mode, + }, + }) +} + +/// Expects ChainMetadata +fn parse_domain(chain: ValueParser, name: &str) -> ConfigResult { + let mut err = ConfigParsingError::default(); + let internal_name = chain.chain(&mut err).get_key("name").parse_string().end(); + + if let Some(internal_name) = internal_name { + if internal_name != name { + Err(eyre!( + "detected chain name mismatch, the config may be corrupted" + )) + } else { + Ok(()) + } + } else { + Err(eyre!("missing chain name, the config may be corrupted")) + } + .take_err(&mut err, || &chain.cwp + "name"); + + let domain_id = chain + .chain(&mut err) + .get_opt_key("domainId") + .parse_u32() + .end() + .or_else(|| chain.chain(&mut err).get_key("chainId").parse_u32().end()); + + let protocol = chain + .chain(&mut err) + .get_key("protocol") + .parse_from_str::("Invalid Hyperlane domain protocol") + .end(); + + cfg_unwrap_all!(&chain.cwp, err: [domain_id, protocol]); + + let domain = HyperlaneDomain::from_config(domain_id, name, protocol) + .context("Invalid domain data") + .take_err(&mut err, || chain.cwp.clone()); + + cfg_unwrap_all!(&chain.cwp, err: [domain]); + err.into_result(domain) +} + +/// Expects AgentSigner. +fn parse_signer(signer: ValueParser) -> ConfigResult { + let mut err = ConfigParsingError::default(); + + let signer_type = signer + .chain(&mut err) + .get_opt_key("signerType") + .parse_string() + .end(); + + let key_is_some = matches!(signer.get_opt_key("key"), Ok(Some(_))); + let id_is_some = matches!(signer.get_opt_key("id"), Ok(Some(_))); + let region_is_some = matches!(signer.get_opt_key("region"), Ok(Some(_))); + + macro_rules! parse_signer { + (hexKey) => {{ + let key = signer + .chain(&mut err) + .get_key("key") + .parse_private_key() + .unwrap_or_default(); + err.into_result(SignerConf::HexKey { key }) + }}; + (aws) => {{ + let id = signer + .chain(&mut err) + .get_key("id") + .parse_string() + .unwrap_or("") + .to_owned(); + let region = signer + .chain(&mut err) + .get_key("region") + .parse_from_str("Expected AWS region") + .unwrap_or_default(); + err.into_result(SignerConf::Aws { id, region }) + }}; + } + + match signer_type { + Some("hexKey") => parse_signer!(hexKey), + Some("aws") => parse_signer!(aws), + Some(t) => { + Err(eyre!("Unknown signer type `{t}`")).into_config_result(|| &signer.cwp + "type") + } + None if key_is_some => parse_signer!(hexKey), + None if id_is_some | region_is_some => parse_signer!(aws), + None => Ok(SignerConf::Node), + } +} + +/// Parser for agent signers. +#[derive(Debug, Deserialize)] +#[serde(transparent)] +pub struct RawAgentSignerConf(Value); + +impl FromRawConf for SignerConf { + fn from_config_filtered( + raw: RawAgentSignerConf, + cwp: &ConfigPath, + _filter: (), + ) -> ConfigResult { + parse_signer(ValueParser::new(cwp.clone(), &raw.0)) + } +} diff --git a/rust/hyperlane-base/src/types/s3_storage.rs b/rust/hyperlane-base/src/types/s3_storage.rs index 544bf6e229..588bc81b76 100644 --- a/rust/hyperlane-base/src/types/s3_storage.rs +++ b/rust/hyperlane-base/src/types/s3_storage.rs @@ -1,10 +1,10 @@ -use std::sync::OnceLock; -use std::{fmt, time::Duration}; +use std::{fmt, sync::OnceLock, time::Duration}; use async_trait::async_trait; use derive_new::new; use eyre::{bail, Result}; use futures_util::TryStreamExt; +use hyperlane_core::{SignedAnnouncement, SignedCheckpoint, SignedCheckpointWithMessageId}; use prometheus::IntGauge; use rusoto_core::{ credential::{Anonymous, AwsCredentials, StaticProvider}, @@ -13,10 +13,7 @@ use rusoto_core::{ use rusoto_s3::{GetObjectError, GetObjectRequest, PutObjectRequest, S3Client, S3}; use tokio::time::timeout; -use hyperlane_core::{SignedAnnouncement, SignedCheckpoint, SignedCheckpointWithMessageId}; - -use crate::settings::aws_credentials::AwsChainCredentialsProvider; -use crate::CheckpointSyncer; +use crate::{settings::aws_credentials::AwsChainCredentialsProvider, CheckpointSyncer}; /// The timeout for S3 requests. Rusoto doesn't offer timeout configuration /// out of the box, so S3 requests must be wrapped with a timeout. diff --git a/rust/hyperlane-core/src/config/mod.rs b/rust/hyperlane-core/src/config/mod.rs index 5d3ffe38c6..58c27d77b9 100644 --- a/rust/hyperlane-core/src/config/mod.rs +++ b/rust/hyperlane-core/src/config/mod.rs @@ -17,9 +17,11 @@ mod trait_ext; /// A result type that is used for config parsing and may contain multiple /// errors. pub type ConfigResult = Result; +/// A no-op filter type. +pub type NoFilter = (); /// A trait that allows for constructing `Self` from a raw config type. -pub trait FromRawConf: Sized +pub trait FromRawConf: Sized where T: Debug, F: Default, @@ -132,10 +134,10 @@ impl std::error::Error for ConfigParsingError {} /// calling this macro a, b, and c will be unwrapped and assigned to variables of the same name. #[macro_export] macro_rules! cfg_unwrap_all { - ($cwp:ident, $err:ident: [$($i:ident),+$(,)?]) => { + ($cwp:expr, $err:ident: [$($i:ident),+$(,)?]) => { $(cfg_unwrap_all!(@unwrap $cwp, $err, $i);)* }; - (@unwrap $cwp:ident, $err:ident, $i:ident) => { + (@unwrap $cwp:expr, $err:ident, $i:ident) => { let $i = if let Some($i) = $i { $i } else { diff --git a/rust/hyperlane-core/src/config/trait_ext.rs b/rust/hyperlane-core/src/config/trait_ext.rs index 7fce68f655..611e3c44f5 100644 --- a/rust/hyperlane-core/src/config/trait_ext.rs +++ b/rust/hyperlane-core/src/config/trait_ext.rs @@ -1,6 +1,7 @@ -use crate::config::{ConfigParsingError, ConfigPath, ConfigResult}; use eyre::Report; +use crate::config::{ConfigParsingError, ConfigPath, ConfigResult}; + /// Extension trait to better support ConfigResults with non-ConfigParsingError /// results. pub trait ConfigErrResultExt { @@ -38,13 +39,18 @@ where } /// Extension trait to better support ConfigResults. -pub trait ConfigResultExt { +pub trait ConfigResultExt { + /// The resulting type + type Output; + /// Take the error from a result and merge it into the given /// ConfigParsingError. - fn take_config_err(self, err: &mut ConfigParsingError) -> Option; + fn take_config_err(self, err: &mut ConfigParsingError) -> Option; } -impl ConfigResultExt for ConfigResult { +impl ConfigResultExt for ConfigResult { + type Output = T; + fn take_config_err(self, err: &mut ConfigParsingError) -> Option { match self { Ok(v) => Some(v), @@ -55,3 +61,21 @@ impl ConfigResultExt for ConfigResult { } } } + +/// Extension trait to better support ConfigResults. +pub trait ConfigResultOptionExt { + /// The resulting type + type Output; + + /// Take the error from a result and merge it into the given + /// ConfigParsingError. + fn take_config_err_flat(self, err: &mut ConfigParsingError) -> Option; +} + +impl ConfigResultOptionExt for ConfigResult> { + type Output = T; + + fn take_config_err_flat(self, err: &mut ConfigParsingError) -> Option { + self.take_config_err(err).flatten() + } +} diff --git a/typescript/sdk/src/metadata/agentConfig.ts b/typescript/sdk/src/metadata/agentConfig.ts index c2d69b8278..1ed8357ba7 100644 --- a/typescript/sdk/src/metadata/agentConfig.ts +++ b/typescript/sdk/src/metadata/agentConfig.ts @@ -105,21 +105,23 @@ export const AgentChainMetadataSchema = ChainMetadataSchema.merge( signer: AgentSignerSchema.optional().describe( 'The signer to use for this chain', ), - index: z.object({ - from: ZUint.optional().describe( - 'The starting block from which to index events.', - ), - chunk: ZNzUint.optional().describe( - 'The number of blocks to index at a time.', - ), - // TODO(2214): I think we can always interpret this from the ProtocolType - mode: z - .nativeEnum(AgentIndexMode) - .optional() - .describe( - 'The indexing method to use for this chain; will attempt to choose a suitable default if not specified.', + index: z + .object({ + from: ZUint.optional().describe( + 'The starting block from which to index events.', ), - }), + chunk: ZNzUint.optional().describe( + 'The number of blocks to index at a time.', + ), + // TODO(2214): I think we can always interpret this from the ProtocolType + mode: z + .nativeEnum(AgentIndexMode) + .optional() + .describe( + 'The indexing method to use for this chain; will attempt to choose a suitable default if not specified.', + ), + }) + .optional(), }); export type AgentChainMetadata = z.infer; @@ -174,25 +176,24 @@ const GasPaymentEnforcementBaseSchema = z.object({ 'An optional matching list, any message that matches will use this policy. By default all messages will match.', ), }); -const GasPaymentEnforcementSchema = z.union([ - GasPaymentEnforcementBaseSchema.extend({ - type: z.literal('none').optional(), - }), - GasPaymentEnforcementBaseSchema.extend({ - type: z.literal('minimum').optional(), - payment: ZUWei, - matchingList: MatchingListSchema.optional().describe( - 'An optional matching list, any message that matches will use this policy. By default all messages will match.', - ), - }), - GasPaymentEnforcementBaseSchema.extend({ - type: z.literal('onChainFeeQuoting'), - gasFraction: z.string().regex(/^\d+ ?\/ ?[1-9]\d*$/), - matchingList: MatchingListSchema.optional().describe( - 'An optional matching list, any message that matches will use this policy. By default all messages will match.', - ), - }), -]); +const GasPaymentEnforcementSchema = z.array( + z.union([ + GasPaymentEnforcementBaseSchema.extend({ + type: z.literal('none').optional(), + }), + GasPaymentEnforcementBaseSchema.extend({ + type: z.literal('minimum').optional(), + payment: ZUWei, + }), + GasPaymentEnforcementBaseSchema.extend({ + type: z.literal('onChainFeeQuoting'), + gasFraction: z + .string() + .regex(/^\d+ ?\/ ?[1-9]\d*$/) + .optional(), + }), + ]), +); export type GasPaymentEnforcement = z.infer; @@ -274,6 +275,13 @@ export const ValidatorAgentConfigSchema = AgentConfigSchema.extend({ type: z.literal('s3'), bucket: z.string().nonempty(), region: z.string().nonempty(), + folder: z + .string() + .nonempty() + .optional() + .describe( + 'The folder/key-prefix to use, defaults to the root of the bucket', + ), }) .describe('A checkpoint syncer that uses S3'), ]),