From 0dc64b4483296f52910b9f3622d56f5f5c319d23 Mon Sep 17 00:00:00 2001 From: Jesse Date: Fri, 22 Dec 2023 13:42:45 +0100 Subject: [PATCH] Various fixes for CLI output (#2262) * Log loaded config files * Remove printing logo * Rename Home directory to Data directory in user-facing messages * Initiating -> Initializing * Simplify * Remove check_update --- dozer-cli/src/cli/helper.rs | 35 ++++-- dozer-cli/src/cli/init.rs | 2 +- dozer-cli/src/config_helper.rs | 12 +- dozer-cli/src/live/state.rs | 2 +- dozer-cli/src/main.rs | 164 ++++----------------------- dozer-cli/src/simple/helper.rs | 2 +- dozer-cli/src/simple/orchestrator.rs | 2 +- dozer-tests/src/e2e_tests/case.rs | 2 +- 8 files changed, 59 insertions(+), 162 deletions(-) diff --git a/dozer-cli/src/cli/helper.rs b/dozer-cli/src/cli/helper.rs index cf32f7e804..510d1e75de 100644 --- a/dozer-cli/src/cli/helper.rs +++ b/dozer-cli/src/cli/helper.rs @@ -11,11 +11,11 @@ use dozer_tracing::LabelsAndProgress; use dozer_types::models::config::default_cache_max_map_size; use dozer_types::prettytable::{row, Table}; use dozer_types::serde_json; +use dozer_types::tracing::info; use dozer_types::{models::config::Config, serde_yaml}; use handlebars::Handlebars; use std::collections::BTreeMap; use std::io::{self, Read}; -use std::path::PathBuf; use std::sync::Arc; use tokio::runtime::Runtime; @@ -24,8 +24,8 @@ pub async fn init_config( config_token: Option, config_overrides: Vec<(String, serde_json::Value)>, ignore_pipe: bool, -) -> Result { - let mut config = load_config(config_paths, config_token, ignore_pipe).await?; +) -> Result<(Config, Vec), CliError> { + let (mut config, loaded_files) = load_config(config_paths, config_token, ignore_pipe).await?; config = apply_overrides(&config, config_overrides)?; @@ -35,7 +35,7 @@ pub async fn init_config( let page_size = page_size::get() as u64; config.cache_max_map_size = Some(cache_max_map_size / page_size * page_size); - Ok(config) + Ok((config, loaded_files)) } pub fn get_base_dir() -> Result { @@ -61,8 +61,10 @@ pub async fn list_sources( ignore_pipe: bool, filter: Option, ) -> Result<(), OrchestrationError> { - let config = init_config(config_paths, config_token, config_overrides, ignore_pipe).await?; + let (config, loaded_files) = + init_config(config_paths, config_token, config_overrides, ignore_pipe).await?; let dozer = init_dozer(runtime, config, Default::default())?; + info!("Loaded config from: {}", loaded_files.join(", ")); let connection_map = dozer.list_connectors().await?; let mut table_parent = Table::new(); for (connection_name, (tables, schemas)) in connection_map { @@ -99,14 +101,17 @@ async fn load_config( config_url_or_paths: Vec, config_token: Option, ignore_pipe: bool, -) -> Result { +) -> Result<(Config, Vec), CliError> { let read_stdin = atty::isnt(Stream::Stdin) && !ignore_pipe; let first_config_path = config_url_or_paths.get(0); match first_config_path { None => Err(ConfigurationFilePathNotProvided), Some(path) => { if path.starts_with("https://") || path.starts_with("http://") { - load_config_from_http_url(path, config_token).await + Ok(( + load_config_from_http_url(path, config_token).await?, + vec![path.to_owned()], + )) } else { load_config_from_file(config_url_or_paths, read_stdin) } @@ -131,21 +136,27 @@ async fn load_config_from_http_url( pub fn load_config_from_file( config_path: Vec, read_stdin: bool, -) -> Result { - let stdin_path = PathBuf::from(""); +) -> Result<(Config, Vec), CliError> { + let stdin_path = ""; let input = if read_stdin { let mut input = String::new(); io::stdin() .read_to_string(&mut input) - .map_err(|e| CannotReadConfig(stdin_path, e))?; + .map_err(|e| CannotReadConfig(stdin_path.into(), e))?; Some(input) } else { None }; - let config_template = combine_config(config_path.clone(), input)?; + let mut loaded_files = Vec::new(); + if input.is_some() { + loaded_files.push(stdin_path.to_owned()); + } + + let (config_template, files) = combine_config(config_path.clone(), input)?; + loaded_files.extend_from_slice(&files); match config_template { - Some(template) => parse_config(&template), + Some(template) => Ok((parse_config(&template)?, loaded_files)), None => Err(FailedToFindConfigurationFiles(config_path.join(", "))), } } diff --git a/dozer-cli/src/cli/init.rs b/dozer-cli/src/cli/init.rs index 2e71e22907..1364745f8c 100644 --- a/dozer-cli/src/cli/init.rs +++ b/dozer-cli/src/cli/init.rs @@ -196,7 +196,7 @@ pub fn generate_config_repl() -> Result<(), OrchestrationError> { }), ), ( - format!("question: Home directory ({:}): ", default_home_dir()), + format!("question: Data directory ({:}): ", default_home_dir()), Box::new(move |(home_dir, config)| { if home_dir.is_empty() { config.home_dir = Some(default_home_dir()); diff --git a/dozer-cli/src/config_helper.rs b/dozer-cli/src/config_helper.rs index 451b8d7ed6..832dcfab14 100644 --- a/dozer-cli/src/config_helper.rs +++ b/dozer-cli/src/config_helper.rs @@ -12,9 +12,10 @@ use glob::glob; pub fn combine_config( config_paths: Vec, stdin_yaml: Option, -) -> Result, ConfigCombineError> { +) -> Result<(Option, Vec), ConfigCombineError> { let mut combined_yaml = serde_yaml::Value::Mapping(Mapping::new()); + let mut loaded_files = Vec::new(); let mut config_found = false; for pattern in config_paths { let files_glob = glob(&pattern).map_err(WrongPatternOfConfigFilesGlob)?; @@ -32,8 +33,8 @@ pub fn combine_config( if name.contains(".yml") || name.contains(".yaml") { config_found = true; } - add_file_content_to_config(&mut combined_yaml, name, content)?; + loaded_files.push(name.to_owned()); } } } @@ -48,11 +49,10 @@ pub fn combine_config( if config_found { // `serde_yaml::from_value` will return deserialization error, not sure why. - serde_yaml::to_string(&combined_yaml) - .map_err(CannotSerializeToString) - .map(Some) + let string = serde_yaml::to_string(&combined_yaml).map_err(CannotSerializeToString)?; + Ok((Some(string), loaded_files)) } else { - Ok(None) + Ok((None, vec![])) } } diff --git a/dozer-cli/src/live/state.rs b/dozer-cli/src/live/state.rs index f7a83baa19..5a566960a7 100644 --- a/dozer-cli/src/live/state.rs +++ b/dozer-cli/src/live/state.rs @@ -131,7 +131,7 @@ impl LiveState { let cli = Cli::parse(); - let config = init_config( + let (config, _) = init_config( cli.config_paths.clone(), cli.config_token.clone(), cli.config_overrides.clone(), diff --git a/dozer-cli/src/main.rs b/dozer-cli/src/main.rs index b207aaec53..deba20ae39 100644 --- a/dozer-cli/src/main.rs +++ b/dozer-cli/src/main.rs @@ -3,25 +3,20 @@ use dozer_api::shutdown; use dozer_cli::cli::cloud::CloudCommands; use dozer_cli::cli::types::{Cli, Commands, ConnectorCommand, RunCommands, SecurityCommands}; use dozer_cli::cli::{generate_config_repl, init_config}; -use dozer_cli::cli::{init_dozer, list_sources, LOGO}; +use dozer_cli::cli::{init_dozer, list_sources}; use dozer_cli::cloud::{cloud_app_context::CloudAppContext, CloudClient, DozerGrpcCloudClient}; use dozer_cli::errors::{CliError, CloudError, OrchestrationError}; use dozer_cli::{live, set_ctrl_handler, set_panic_hook}; use dozer_tracing::LabelsAndProgress; use dozer_types::models::config::Config; use dozer_types::models::telemetry::{TelemetryConfig, TelemetryMetricsConfig}; -use dozer_types::serde::Deserialize; use dozer_types::tracing::{error, error_span, info}; use futures::stream::{AbortHandle, Abortable}; -use std::cmp::Ordering; use std::convert::identity; use std::sync::Arc; use tokio::runtime::Runtime; -use tokio::time; -use dozer_types::log::{debug, warn}; -use std::time::Duration; -use std::{env, process}; +use std::process; fn main() { if let Err(e) = run() { @@ -30,94 +25,6 @@ fn main() { } } -fn render_logo() { - const VERSION: &str = env!("CARGO_PKG_VERSION"); - - println!("{LOGO}"); - println!("\nDozer Version: {VERSION}\n"); -} - -#[derive(Deserialize, Debug)] -#[serde(crate = "dozer_types::serde")] -struct DozerPackage { - #[serde(rename(deserialize = "latestVersion"))] - pub latest_version: String, - #[serde(rename(deserialize = "availableAssets"))] - pub _available_assets: Vec, - pub link: String, -} - -fn version_to_vector(version: &str) -> Vec { - version.split('.').map(|s| s.parse().unwrap()).collect() -} - -fn compare_versions(v1: Vec, v2: Vec) -> bool { - for i in 0..v1.len() { - match v1.get(i).cmp(&v2.get(i)) { - Ordering::Greater => return true, - Ordering::Less => return false, - Ordering::Equal => continue, - } - } - false -} - -async fn check_update() { - const VERSION: &str = env!("CARGO_PKG_VERSION"); - let dozer_env = std::env::var("DOZER_ENV").unwrap_or("local".to_string()); - let dozer_dev = std::env::var("DOZER_DEV").unwrap_or("ext".to_string()); - let query = vec![ - ("version", VERSION), - ("build", std::env::consts::ARCH), - ("os", std::env::consts::OS), - ("env", &dozer_env), - ("dev", &dozer_dev), - ]; - - let request_url = "https://metadata.dev.getdozer.io/"; - - let client = reqwest::Client::new(); - - let mut printed = false; - - loop { - let response = client - .get(&request_url.to_string()) - .query(&query) - .send() - .await; - - match response { - Ok(r) => { - if !printed { - let package: DozerPackage = r.json().await.unwrap(); - let current = version_to_vector(VERSION); - let remote = version_to_vector(&package.latest_version); - - if compare_versions(remote, current) { - info!("A new version of Dozer is available."); - info!( - "You can download v{}, from {}.", - package.latest_version, package.link - ); - printed = true; - } - } - } - Err(e) => { - // We dont show error if error is connection error, because mostly it happens - // when main thread is shutting down before request completes. - if !e.is_connect() { - warn!("Unable to fetch the latest metadata"); - } - - debug!("Updates check error: {}", e); - } - } - time::sleep(Duration::from_secs(2 * 60 * 60)).await; - } -} - fn run() -> Result<(), OrchestrationError> { // Reloading trace layer seems impossible, so we are running Cli::parse in a closure // and then initializing it after reading the configuration. This is a hacky workaround, but it works. @@ -136,7 +43,7 @@ fn run() -> Result<(), OrchestrationError> { // Now we have access to telemetry configuration. Telemetry must be initialized in tokio runtime. let app_id = config_res .as_ref() - .map(|c| c.cloud.app_id.as_deref().unwrap_or(&c.app_name)) + .map(|(c, _)| c.cloud.app_id.as_deref().unwrap_or(&c.app_name)) .ok(); // We always enable telemetry when running live. @@ -148,7 +55,7 @@ fn run() -> Result<(), OrchestrationError> { } else { config_res .as_ref() - .map(|c| c.telemetry.clone()) + .map(|(c, _)| c.telemetry.clone()) .unwrap_or_default() }; @@ -158,7 +65,8 @@ fn run() -> Result<(), OrchestrationError> { if let Commands::Cloud(cloud) = &cli.cmd { return run_cloud(cloud, runtime, &cli); } - let config = config_res?; + let (config, config_files) = config_res?; + info!("Loaded config from: {}", config_files.join(", ")); let dozer = init_dozer( runtime.clone(), @@ -170,26 +78,16 @@ fn run() -> Result<(), OrchestrationError> { // run individual servers (match cli.cmd { Commands::Run(run) => match run.command { - Some(RunCommands::Api) => { - render_logo(); - dozer.runtime.block_on(dozer.run_api(shutdown_receiver)) - } - Some(RunCommands::App) => { - render_logo(); - dozer - .runtime - .block_on(dozer.run_apps(shutdown_receiver, None)) - } + Some(RunCommands::Api) => dozer.runtime.block_on(dozer.run_api(shutdown_receiver)), + Some(RunCommands::App) => dozer + .runtime + .block_on(dozer.run_apps(shutdown_receiver, None)), Some(RunCommands::Lambda) => { - render_logo(); dozer.runtime.block_on(dozer.run_lambda(shutdown_receiver)) } - None => { - render_logo(); - dozer - .runtime - .block_on(dozer.run_all(shutdown_receiver, run.locked)) - } + None => dozer + .runtime + .block_on(dozer.run_all(shutdown_receiver, run.locked)), }, Commands::Security(security) => match security.command { SecurityCommands::GenerateToken => { @@ -235,7 +133,6 @@ fn run() -> Result<(), OrchestrationError> { panic!("This should not happen as it is handled in parse_and_generate"); } Commands::Live(live_flags) => { - render_logo(); dozer.runtime.block_on(live::start_live_server( &dozer.runtime, shutdown_receiver, @@ -256,10 +153,11 @@ fn run_cloud( runtime: Arc, cli: &Cli, ) -> Result<(), OrchestrationError> { - render_logo(); let cloud = cloud.clone(); - let config = init_configuration(cli, runtime.clone()).ok(); + let config = init_configuration(cli, runtime.clone()) + .ok() + .map(|(config, _)| config); let mut cloud_client = CloudClient::new(cloud.clone(), config.clone(), runtime.clone()); match cloud.command.clone() { CloudCommands::Deploy(deploy) => cloud_client.deploy(deploy, cli.config_paths.clone()), @@ -312,27 +210,15 @@ fn parse_and_generate() -> Result { ) } -fn init_configuration(cli: &Cli, runtime: Arc) -> Result { - dozer_tracing::init_telemetry_closure( - None, - &Default::default(), - || -> Result { - let res = runtime.block_on(init_config( - cli.config_paths.clone(), - cli.config_token.clone(), - cli.config_overrides.clone(), - cli.ignore_pipe, - )); - - match res { - Ok(config) => { - runtime.spawn(check_update()); - Ok(config) - } - Err(e) => Err(e), - } - }, - ) +fn init_configuration(cli: &Cli, runtime: Arc) -> Result<(Config, Vec), CliError> { + dozer_tracing::init_telemetry_closure(None, &Default::default(), || -> Result<_, CliError> { + runtime.block_on(init_config( + cli.config_paths.clone(), + cli.config_token.clone(), + cli.config_overrides.clone(), + cli.ignore_pipe, + )) + }) } fn display_error(e: &OrchestrationError) { diff --git a/dozer-cli/src/simple/helper.rs b/dozer-cli/src/simple/helper.rs index fae9cd86e6..f6b813c014 100644 --- a/dozer-cli/src/simple/helper.rs +++ b/dozer-cli/src/simple/helper.rs @@ -10,7 +10,7 @@ use dozer_types::prettytable::{row, Table}; pub fn validate_config(config: &Config) -> Result<(), OrchestrationError> { info!( - "Home dir: {}", + "Data directory: {}", get_colored_text( &config.home_dir.clone().unwrap_or_else(default_home_dir), PURPLE diff --git a/dozer-cli/src/simple/orchestrator.rs b/dozer-cli/src/simple/orchestrator.rs index e4e71e913b..2d2602ed93 100644 --- a/dozer-cli/src/simple/orchestrator.rs +++ b/dozer-cli/src/simple/orchestrator.rs @@ -359,7 +359,7 @@ impl SimpleOrchestrator { let home_dir = HomeDir::new(home_dir, cache_dir); info!( - "Initiating app: {}", + "Initializing app: {}", get_colored_text(&self.config.app_name, PURPLE) ); if force { diff --git a/dozer-tests/src/e2e_tests/case.rs b/dozer-tests/src/e2e_tests/case.rs index db4e39b8ed..eeeaf38401 100644 --- a/dozer-tests/src/e2e_tests/case.rs +++ b/dozer-tests/src/e2e_tests/case.rs @@ -37,7 +37,7 @@ pub struct Case { impl Case { pub fn load_from_case_dir(case_dir: PathBuf, connections_dir: PathBuf) -> Self { let dozer_config_path = find_dozer_config_path(&case_dir); - let dozer_config = load_config_from_file(vec![dozer_config_path.clone()], false) + let (dozer_config, _) = load_config_from_file(vec![dozer_config_path.clone()], false) .unwrap_or_else(|e| panic!("Cannot read file: {}: {:?}", &dozer_config_path, e)); let mut connections = HashMap::new(); for connection in &dozer_config.connections {