Improve: Impled unit tests and features

.github/workflows/pull-requests.yml

@@ -1,9 +1,17 @@
 name: Pull Request Actions
 
 on:
+  push:
+    branches:
+      - master
+
   pull_request:
     branches:
       - master
+    types:
+      - opened
+      - reopened
+      - synchronize
 
 env:
   CARGO_TERM_COLOR: always
@@ -20,8 +28,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Test
-        run: git status
       - name: Fmt
         run: cargo fmt --all --verbose --check
 
@@ -43,3 +49,23 @@ jobs:
       - uses: actions/checkout@v4
       - name: Test
         run: cargo test --all --verbose
+
+  build-platforms:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust-target: [
+          'x86_64-unknown-linux-gnu',
+          'aarch64-apple-darwin',
+          'x86_64-apple-darwin'
+        ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          target: ${{ matrix.rust-target }}
+      - name: Add rustup target ${{ matrix.rust-target }}
+        run: rustup target add ${{ matrix.rust-target }}
+      - name: Build app for ${{ matrix.rust-target }}
+        run: cargo build --release --target ${{ matrix.rust-target }} Cargo.toml

Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 maintenance = { status = "actively-developed" }
 
 [features]
+enable-cacher = ["dep:redis"]
 default = []
 
 [dependencies]
@@ -37,7 +38,8 @@ features = ["rustc-serialize", "serde"]
 
 [dependencies.redis]
 version = "0.27.3"
-features = ["aio", "tokio-comp", "connection-manager"]
+features = ["aio", "tokio-comp", "connection-manager", "serde_json", "json"]
+optional = true
 
 [dependencies.reqwest]
 version = "0.12.8"

config/development.toml

@@ -12,8 +12,8 @@ allowed = "*"
 max_age = 3600
 
 [elastic]
-address = "localhost:9200"
-enabled_tls = "false"
+address = "130.193.37.41:9200"
+enabled_tls = "true"
 username = "elastic"
 password = "elastic"
 

crates/elquery/src/filter/must_filter.rs

@@ -1,6 +1,7 @@
 use serde_derive::Serialize;
 use serde_json::{json, Value};
 
+#[allow(dead_code)]
 trait MustFilterItemTrait {}
 impl MustFilterItemTrait for TermFilterItem {}
 impl MustFilterItemTrait for RangeFilterItem {}

src/bin/doc-searcher-run.rs

@@ -2,7 +2,6 @@ extern crate doc_search;
 
 use actix_web::middleware::Logger;
 use actix_web::{web, App, HttpServer};
-use doc_search::cacher::redis::RedisClient;
 use doc_search::cors::build_cors;
 use doc_search::elastic::ElasticClient;
 use doc_search::embeddings::native::EmbeddingsClient;
@@ -15,6 +14,9 @@ use doc_search::storage::endpoints::build_scope as build_storage_scope;
 use doc_search::storage::{DocumentService, FolderService};
 use doc_search::{config, swagger, Connectable};
 
+#[cfg(feature = "enable-cacher")]
+use doc_search::cacher;
+
 #[actix_web::main]
 async fn main() -> Result<(), anyhow::Error> {
     let s_config = config::ServiceConfig::new()?;
@@ -24,29 +26,39 @@ async fn main() -> Result<(), anyhow::Error> {
     let logger_config = s_config.logger();
     init_logger(logger_config)?;
 
-    let cacher_service = RedisClient::connect(s_config.cacher())?;
     let search_service = ElasticClient::connect(s_config.elastic())?;
     let embeddings_service = EmbeddingsClient::connect(s_config.embeddings())?;
 
+    #[cfg(feature = "enable-cacher")]
+    let cacher_service = cacher::redis::RedisClient::connect(s_config.cacher())?;
+
     HttpServer::new(move || {
         let cors = build_cors(&cors_config.clone());
         let logger = Logger::default();
-        let cacher_cxt = Box::new(cacher_service.clone());
 
         let documents_cxt: Box<dyn DocumentService> = Box::new(search_service.clone());
         let folders_cxt: Box<dyn FolderService> = Box::new(search_service.clone());
         let paginator_cxt: Box<dyn PaginatorService> = Box::new(search_service.clone());
         let searcher_cxt: Box<dyn SearcherService> = Box::new(search_service.clone());
         let embeddings_cxt: Box<dyn EmbeddingsService> = Box::new(embeddings_service.clone());
 
-        App::new()
+        let app = App::new()
             .app_data(web::Data::new(documents_cxt))
             .app_data(web::Data::new(folders_cxt))
             .app_data(web::Data::new(paginator_cxt))
-            .app_data(web::Data::new(cacher_cxt))
             .app_data(web::Data::new(searcher_cxt))
-            .app_data(web::Data::new(embeddings_cxt))
-            .wrap(logger)
+            .app_data(web::Data::new(embeddings_cxt));
+
+        #[cfg(feature = "enable-cacher")]
+        let cacher_search_cxt: cacher::redis::SearchParamsCached = Box::new(cacher_service.clone());
+        #[cfg(feature = "enable-cacher")]
+        let cacher_paginate_cxt: cacher::redis::PaginatedCached = Box::new(cacher_service.clone());
+        #[cfg(feature = "enable-cacher")]
+        let app = app
+            .app_data(web::Data::new(cacher_search_cxt))
+            .app_data(web::Data::new(cacher_paginate_cxt));
+
+        app.wrap(logger)
             .wrap(cors)
             .service(build_metrics_scope())
             .service(build_storage_scope())

src/cacher/mod.rs

@@ -1,4 +1,6 @@
 pub mod config;
+
+#[cfg(feature = "enable-cacher")]
 pub mod redis;
 
 #[async_trait::async_trait]

src/cacher/redis/mod.rs

@@ -1,17 +1,23 @@
+mod models;
+
 use crate::cacher::config::CacherConfig;
 use crate::cacher::CacherService;
+use crate::searcher::forms::PaginateNextForm;
+use crate::searcher::models::{Paginated, SearchParams};
 use crate::Connectable;
 
 use getset::CopyGetters;
 use redis::{AsyncCommands, Client, RedisError, RedisResult};
+use serde_json::Value;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 
+pub type SearchParamsCached = Box<dyn CacherService<SearchParams, Paginated<Vec<Value>>>>;
+pub type PaginatedCached = Box<dyn CacherService<PaginateNextForm, Paginated<Vec<Value>>>>;
+
 #[derive(Clone, CopyGetters)]
 pub struct RedisClient {
-    // #[getset(get_copy = "pub")]
     options: Arc<RedisOptions>,
-    // #[getset(get_copy = "pub")]
     client: Arc<RwLock<Client>>,
 }
 

src/cacher/redis/models.rs

@@ -0,0 +1,91 @@
+use crate::searcher::forms::PaginateNextForm;
+use crate::searcher::models::{Paginated, SearchParams};
+use crate::storage::models::Document;
+
+use redis::{RedisError, RedisResult, RedisWrite, Value};
+use serde::ser::Error;
+
+impl redis::ToRedisArgs for PaginateNextForm {
+    fn write_redis_args<W>(&self, out: &mut W)
+    where
+        W: ?Sized + RedisWrite,
+    {
+        match serde_json::to_string(self) {
+            Ok(json_str) => out.write_arg_fmt(json_str),
+            Err(err) => {
+                tracing::error!("cacher: failed to serialize paginate form: {err:#?}");
+            }
+        }
+    }
+}
+
+impl redis::ToRedisArgs for SearchParams {
+    fn write_redis_args<W>(&self, out: &mut W)
+    where
+        W: ?Sized + RedisWrite,
+    {
+        match serde_json::to_string(self) {
+            Ok(json_str) => out.write_arg_fmt(json_str),
+            Err(err) => {
+                tracing::error!("cacher: failed to serialize search parameters: {err:#?}");
+            }
+        }
+    }
+}
+
+impl redis::ToRedisArgs for Document {
+    fn write_redis_args<W>(&self, out: &mut W)
+    where
+        W: ?Sized + RedisWrite,
+    {
+        match serde_json::to_string(self) {
+            Ok(json_str) => out.write_arg_fmt(json_str),
+            Err(err) => {
+                tracing::error!("cacher: failed to serialize document: {err:#?}");
+            }
+        }
+    }
+}
+
+impl redis::ToRedisArgs for Paginated<Vec<serde_json::Value>> {
+    fn write_redis_args<W>(&self, out: &mut W)
+    where
+        W: ?Sized + RedisWrite,
+    {
+        match serde_json::to_string(self) {
+            Ok(json_str) => out.write_arg_fmt(json_str),
+            Err(err) => {
+                tracing::error!("cacher: failed to serialize paginated docs: {err:#?}");
+            }
+        }
+    }
+}
+
+impl redis::FromRedisValue for Document {
+    fn from_redis_value(v: &Value) -> RedisResult<Self> {
+        match v {
+            Value::BulkString(data) => {
+                serde_json::from_slice::<Document>(data.as_slice()).map_err(RedisError::from)
+            }
+            _ => {
+                let err = serde_json::Error::custom("failed to extract redis value type");
+                Err(RedisError::from(err))
+            }
+        }
+    }
+}
+
+impl redis::FromRedisValue for Paginated<Vec<serde_json::Value>> {
+    fn from_redis_value(v: &Value) -> RedisResult<Self> {
+        match v {
+            Value::BulkString(data) => {
+                serde_json::from_slice::<Paginated<Vec<serde_json::Value>>>(data.as_slice())
+                    .map_err(RedisError::from)
+            }
+            _ => {
+                let err = serde_json::Error::custom("failed to extract redis value type");
+                Err(RedisError::from(err))
+            }
+        }
+    }
+}

src/searcher/elastic/extractor.rs

@@ -204,76 +204,3 @@ impl SearcherTrait<InfoFolder> for InfoFolder {
         InfoFolder::deserialize(source_value).map_err(WebError::from)
     }
 }
-
-#[cfg(test)]
-mod test_all_search_queries {
-    use super::*;
-
-    #[tokio::test]
-    async fn test_document_build_query() -> Result<(), anyhow::Error> {
-        let s_params = build_search_params();
-        let build_query = Document::build_query(&s_params).await;
-        println!("{}", serde_json::to_string_pretty(&build_query).unwrap());
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_document_preview_build_query() -> Result<(), anyhow::Error> {
-        let s_params = build_search_params();
-        let build_query = DocumentPreview::build_query(&s_params).await;
-        println!("{}", serde_json::to_string_pretty(&build_query).unwrap());
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_document_vectors_build_query() -> Result<(), anyhow::Error> {
-        let s_params = build_search_params();
-        let build_query = DocumentVectors::build_query(&s_params).await;
-        println!("{}", serde_json::to_string_pretty(&build_query).unwrap());
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_info_folder_system_build_query() -> Result<(), anyhow::Error> {
-        let s_params = build_search_params();
-        let build_query = InfoFolder::build_query(&s_params).await;
-        println!("{}", serde_json::to_string_pretty(&build_query).unwrap());
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_info_folder_build_query() -> Result<(), anyhow::Error> {
-        let mut s_params = build_search_params();
-        s_params.set_show_all(false);
-
-        let build_query = InfoFolder::build_query(&s_params).await;
-        println!("{}", serde_json::to_string_pretty(&build_query).unwrap());
-
-        Ok(())
-    }
-
-    fn build_search_params() -> SearchParams {
-        SearchParams::builder()
-            .query("Some query".to_string())
-            .query_tokens(Some(Vec::default()))
-            .folder_ids(Some("test-folder-id".to_string()))
-            .document_type("document".to_string())
-            .document_extension("txt".to_string())
-            .created_date_to("2025-04-26T11:14:55Z".to_string())
-            .created_date_from("2024-04-26T11:14:55Z".to_string())
-            .document_size_to(37000)
-            .document_size_from(0)
-            .result_size(25)
-            .result_offset(0)
-            .scroll_lifetime("1m".to_string())
-            .knn_amount(Some(5))
-            .knn_candidates(Some(100))
-            .show_all(Some(true))
-            .build()
-            .unwrap()
-    }
-}

src/searcher/elastic/helper.rs

@@ -118,7 +118,7 @@ pub async fn search<T>(
     indexes: &[&str],
 ) -> Result<Paginated<Vec<T>>, WebError>
 where
-    T: DocumentsTrait + SearcherTrait<T>,
+    T: DocumentsTrait + SearcherTrait<T> + serde::Serialize,
 {
     let body_value = T::build_query(s_params).await;
     let response = send_search_request(elastic, s_params, &body_value, indexes).await?;
@@ -137,7 +137,7 @@ pub async fn search_all<T>(
     indexes: &[&str],
 ) -> Result<Paginated<Vec<T>>, WebError>
 where
-    T: DocumentsTrait + SearcherTrait<T>,
+    T: DocumentsTrait + SearcherTrait<T> + serde::Serialize,
 {
     let body_value = DocumentPreview::build_query(s_params).await;
     let response = send_search_request(elastic, s_params, &body_value, indexes).await?;
@@ -171,7 +171,7 @@ pub async fn send_search_request(
 
 pub async fn extract_elastic_response<T>(response: Response) -> Paginated<Vec<T>>
 where
-    T: DocumentsTrait + SearcherTrait<T>,
+    T: DocumentsTrait + SearcherTrait<T> + serde::Serialize,
 {
     let common_object = response.json::<Value>().await.unwrap();
     let document_json = &common_object[&"hits"][&"hits"];