From 5dcd5d87974a87c8a1803631a1d27360f17cbe6f Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 15 Jan 2025 16:40:38 +0100 Subject: [PATCH] Introduce a new experimental document compression parameter --- .../meilisearch/src/analytics/segment_analytics.rs | 3 +++ crates/meilisearch/src/option.rs | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/crates/meilisearch/src/analytics/segment_analytics.rs b/crates/meilisearch/src/analytics/segment_analytics.rs index a97813089..4b1bdd757 100644 --- a/crates/meilisearch/src/analytics/segment_analytics.rs +++ b/crates/meilisearch/src/analytics/segment_analytics.rs @@ -194,6 +194,7 @@ struct Infos { experimental_reduce_indexing_memory_usage: bool, experimental_max_number_of_batched_tasks: usize, experimental_limit_batched_tasks_total_size: u64, + experimental_enable_document_compression: bool, gpu_enabled: bool, db_path: bool, import_dump: bool, @@ -240,6 +241,7 @@ impl Infos { experimental_reduce_indexing_memory_usage, experimental_max_number_of_batched_tasks, experimental_limit_batched_tasks_total_size, + experimental_enable_document_compression, http_addr, master_key: _, env, @@ -299,6 +301,7 @@ impl Infos { experimental_replication_parameters, experimental_enable_logs_route: experimental_enable_logs_route | logs_route, experimental_reduce_indexing_memory_usage, + experimental_enable_document_compression, gpu_enabled: meilisearch_types::milli::vector::is_cuda_enabled(), db_path: db_path != PathBuf::from("./data.ms"), import_dump: import_dump.is_some(), diff --git a/crates/meilisearch/src/option.rs b/crates/meilisearch/src/option.rs index b5aa6b9e7..e71121f57 100644 --- a/crates/meilisearch/src/option.rs +++ b/crates/meilisearch/src/option.rs @@ -62,6 +62,8 @@ const MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS: &str = "MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS"; const MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS_TOTAL_SIZE: &str = "MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS_SIZE"; +const MEILI_EXPERIMENTAL_ENABLE_DOCUMENT_COMPRESSION: &str = + "MEILI_EXPERIMENTAL_ENABLE_DOCUMENT_COMPRESSION"; const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml"; const DEFAULT_DB_PATH: &str = "./data.ms"; @@ -438,6 +440,11 @@ pub struct Opt { #[serde(default = "default_limit_batched_tasks_total_size")] pub experimental_limit_batched_tasks_total_size: u64, + /// Experimentally enable the document compression feature, see: + #[clap(long, env = MEILI_EXPERIMENTAL_ENABLE_DOCUMENT_COMPRESSION)] + #[serde(default)] + pub experimental_enable_document_compression: bool, + #[serde(flatten)] #[clap(flatten)] pub indexer_options: IndexerOpts, @@ -540,6 +547,7 @@ impl Opt { experimental_reduce_indexing_memory_usage, experimental_max_number_of_batched_tasks, experimental_limit_batched_tasks_total_size, + experimental_enable_document_compression, } = self; export_to_env_if_not_present(MEILI_DB_PATH, db_path); export_to_env_if_not_present(MEILI_HTTP_ADDR, http_addr); @@ -628,6 +636,10 @@ impl Opt { MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS_TOTAL_SIZE, experimental_limit_batched_tasks_total_size.to_string(), ); + export_to_env_if_not_present( + MEILI_EXPERIMENTAL_ENABLE_DOCUMENT_COMPRESSION, + experimental_enable_document_compression.to_string(), + ); indexer_options.export_to_env(); }