From cb0c890b218045dd54db5fb24c9a00626c8c61ee Mon Sep 17 00:00:00 2001 From: Bread White <32078281+breadrock1@users.noreply.github.com> Date: Mon, 28 Oct 2024 00:43:06 +0300 Subject: [PATCH] Feature: Impl prometheus metrics (#44) * refactor(project): Save point of project and project structure refactoring * fix(tests): Fixed tests after all changes * feat(prometheus): Impled prometheus metrics sending * chore(docker): Added prometheus to docker-compose file * chore(fmt): Applied fmt for new changes --- Cargo.toml | 6 ++++++ config/development.toml | 2 +- docker-compose.yml | 18 ++++++++++++++++++ prometheus/prometheus.yml | 18 ++++++++++++++++++ src/bin/doc-searcher-run.rs | 14 +++++++++++++- src/config.rs | 9 ++------- src/metrics/endpoints.rs | 20 +++++++++++++++----- src/metrics/mod.rs | 3 +++ src/metrics/prometheus.rs | 14 ++++++++++++++ src/swagger/mod.rs | 2 +- 10 files changed, 91 insertions(+), 15 deletions(-) create mode 100644 prometheus/prometheus.yml create mode 100644 src/metrics/prometheus.rs diff --git a/Cargo.toml b/Cargo.toml index 066cf33..79b5faf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ maintenance = { status = "actively-developed" } [features] enable-cacher = ["dep:redis"] enable-semantic = [] +enable-prometheus = ["dep:actix-web-prom"] default = [] [dependencies] @@ -38,6 +39,11 @@ tracing-actix-web = "^0.7" version = "^0.4" features = ["rustc-serialize", "serde"] +[dependencies.actix-web-prom] +optional = true +version = "^0.9" +features = ["process"] + [dependencies.redis] optional = true version = "0.27.3" diff --git a/config/development.toml b/config/development.toml index 8d2b72a..4485b4e 100644 --- a/config/development.toml +++ b/config/development.toml @@ -12,7 +12,7 @@ allowed = "*" max_age = 3600 [elastic] -address = "localhost:9200" +address = "158.160.44.99:9200" enabled_tls = "true" username = "elastic" password = "elastic" diff --git a/docker-compose.yml b/docker-compose.yml index 3fbc6d1..78015e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,22 @@ services: REDIS_PASSWORD: ${REDIS_ROOT_PASSWORD} REDIS_USER_PASSWORD: ${REDIS_CLIENT_PASSWORD} + prometheus: + image: prom/prometheus:v2.36.2 + restart: on-failure + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + links: + - doc-searcher + ports: + - '9090:9090' + volumes: + - './prometheus/:/etc/prometheus/' + - 'prom-vol-1:/prometheus' + doc-searcher: image: doc-searcher:latest command: /app/doc-searcher-init && /app/doc-searcher-run @@ -64,3 +80,5 @@ volumes: driver: local cacher-vol-1: driver: local + prom-vol-1: + driver: local diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..2a8a7a3 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + + external_labels: + monitor: 'doc-searcher' + +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 15s + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'doc-searcher' + metrics_path: '/metrics' + scrape_interval: 15s + static_configs: + - targets: ['doc-searcher:2892'] diff --git a/src/bin/doc-searcher-run.rs b/src/bin/doc-searcher-run.rs index 864c38c..1f1e29a 100644 --- a/src/bin/doc-searcher-run.rs +++ b/src/bin/doc-searcher-run.rs @@ -16,6 +16,9 @@ use doc_search::cacher; #[cfg(feature = "enable-semantic")] use doc_search::embeddings; +#[cfg(feature = "enable-prometheus")] +use doc_search::metrics::prometheus; + #[actix_web::main] async fn main() -> Result<(), anyhow::Error> { let s_config = config::ServiceConfig::new()?; @@ -25,7 +28,8 @@ async fn main() -> Result<(), anyhow::Error> { let logger_config = s_config.logger(); logger::init_logger(logger_config)?; - let search_service = elastic::ElasticClient::connect(s_config.elastic())?; + #[cfg(feature = "enable-prometheus")] + let prometheus = prometheus::init_prometheus()?; #[cfg(feature = "enable-semantic")] let embed_service = embeddings::native::EmbeddingsClient::connect(s_config.embeddings())?; @@ -33,6 +37,8 @@ async fn main() -> Result<(), anyhow::Error> { #[cfg(feature = "enable-cacher")] let cacher_service = cacher::redis::RedisClient::connect(s_config.cacher())?; + let search_service = elastic::ElasticClient::connect(s_config.elastic())?; + HttpServer::new(move || { let cors = cors::build_cors(&cors_config.clone()); let logger = Logger::default(); @@ -56,17 +62,23 @@ async fn main() -> Result<(), anyhow::Error> { #[cfg(feature = "enable-cacher")] let cacher_search_cxt: cacher::redis::SemanticParamsCached = Box::new(cacher_service.clone()); + #[cfg(feature = "enable-cacher")] let cacher_fulltext_cxt: cacher::redis::FullTextParamsCached = Box::new(cacher_service.clone()); + #[cfg(feature = "enable-cacher")] let cacher_paginate_cxt: cacher::redis::PaginatedCached = Box::new(cacher_service.clone()); + #[cfg(feature = "enable-cacher")] let app = app .app_data(web::Data::new(cacher_search_cxt)) .app_data(web::Data::new(cacher_fulltext_cxt)) .app_data(web::Data::new(cacher_paginate_cxt)); + #[cfg(feature = "enable-prometheus")] + let app = app.wrap(prometheus.clone()); + app.wrap(logger) .wrap(cors) .service(build_metrics_scope()) diff --git a/src/config.rs b/src/config.rs index 03fef0d..606e64c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -9,19 +9,14 @@ use derive_builder::Builder; use getset::{CopyGetters, Getters}; use serde_derive::Deserialize; -#[derive(Builder, Clone, Deserialize, CopyGetters, Getters)] +#[derive(Builder, Clone, Deserialize, Getters)] +#[getset(get = "pub")] pub struct ServiceConfig { - #[getset(get = "pub")] logger: LoggerConfig, - #[getset(get = "pub")] server: ServerConfig, - #[getset(get = "pub")] cors: CorsConfig, - #[getset(get = "pub")] elastic: ElasticConfig, - #[getset(get = "pub")] cacher: CacherConfig, - #[getset(get = "pub")] embeddings: EmbeddingsConfig, } diff --git a/src/metrics/endpoints.rs b/src/metrics/endpoints.rs index d5fc758..1830042 100644 --- a/src/metrics/endpoints.rs +++ b/src/metrics/endpoints.rs @@ -1,15 +1,20 @@ use crate::errors::{ErrorResponse, JsonResponse, Successful}; use actix_web::web::Json; -use actix_web::{get, web, Scope}; +use actix_web::{get, web, HttpResponse, Scope}; pub fn build_scope() -> Scope { - web::scope("/metrics").service(metrics) + let scope = web::scope("/metrics").service(hello); + + #[cfg(feature = "enable-prometheus")] + let scope = scope.service(metrics); + + scope } #[utoipa::path( get, - path = "/metrics/", + path = "/metrics/hello", tag = "Metrics", responses( ( @@ -26,7 +31,12 @@ pub fn build_scope() -> Scope { ), ), )] -#[get("/")] -async fn metrics() -> JsonResponse { +#[get("/hello")] +async fn hello() -> JsonResponse { Ok(Json(Successful::new(200, "Ok"))) } + +#[get("/metrics")] +async fn metrics() -> HttpResponse { + HttpResponse::Ok().finish() +} diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index c4b360f..7f78585 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -1 +1,4 @@ pub mod endpoints; + +#[cfg(feature = "enable-prometheus")] +pub mod prometheus; diff --git a/src/metrics/prometheus.rs b/src/metrics/prometheus.rs new file mode 100644 index 0000000..5cf19b8 --- /dev/null +++ b/src/metrics/prometheus.rs @@ -0,0 +1,14 @@ +use actix_web_prom::{PrometheusMetrics, PrometheusMetricsBuilder}; +use std::collections::HashMap; + +pub fn init_prometheus() -> Result { + let mut labels = HashMap::new(); + labels.insert("label1".to_string(), "value1".to_string()); + let prometheus = PrometheusMetricsBuilder::new("metrics") + .endpoint("/metrics") + .const_labels(labels) + .build() + .map_err(|err| anyhow::Error::msg(err.to_string()))?; + + Ok(prometheus) +} diff --git a/src/swagger/mod.rs b/src/swagger/mod.rs index e8281d4..58e9ec2 100644 --- a/src/swagger/mod.rs +++ b/src/swagger/mod.rs @@ -21,7 +21,7 @@ use utoipa_swagger_ui::SwaggerUi; description = "There is API endpoints of DocSearch project based on Rust and Elasticsearch technologies." ), paths( - metrics, + hello, get_folder, get_folders, create_folder,