From 99fec2778832de43bc3a784813011c59f4eff278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Dec 2023 10:55:33 +0100 Subject: [PATCH] Make the --max-number-of-batched-tasks argument experimental --- .../src/analytics/segment_analytics.rs | 6 +++--- meilisearch/src/lib.rs | 2 +- meilisearch/src/option.rs | 19 ++++++++++--------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 54f5813f0..f75516731 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -251,6 +251,7 @@ struct Infos { env: String, experimental_enable_metrics: bool, experimental_reduce_indexing_memory_usage: bool, + experimental_max_number_of_batched_tasks: usize, db_path: bool, import_dump: bool, dump_dir: bool, @@ -263,7 +264,6 @@ struct Infos { ignore_snapshot_if_db_exists: bool, http_addr: bool, http_payload_size_limit: Byte, - max_number_of_batched_tasks: usize, log_level: String, max_indexing_memory: MaxMemory, max_indexing_threads: MaxThreads, @@ -286,13 +286,13 @@ impl From for Infos { db_path, experimental_enable_metrics, experimental_reduce_indexing_memory_usage, + experimental_max_number_of_batched_tasks, http_addr, master_key: _, env, max_index_size: _, max_task_db_size: _, http_payload_size_limit, - max_number_of_batched_tasks, ssl_cert_path, ssl_key_path, ssl_auth_path, @@ -342,7 +342,7 @@ impl From for Infos { ignore_snapshot_if_db_exists, http_addr: http_addr != default_http_addr(), http_payload_size_limit, - max_number_of_batched_tasks, + experimental_max_number_of_batched_tasks, log_level: log_level.to_string(), max_indexing_memory, max_indexing_threads, diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index 896375108..e0f488eea 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -234,7 +234,7 @@ fn open_or_create_database_unchecked( indexer_config: (&opt.indexer_options).try_into()?, autobatching_enabled: true, max_number_of_tasks: 1_000_000, - max_number_of_batched_tasks: opt.max_number_of_batched_tasks, + max_number_of_batched_tasks: opt.experimental_max_number_of_batched_tasks, index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize, index_count: DEFAULT_INDEX_COUNT, instance_features, diff --git a/meilisearch/src/option.rs b/meilisearch/src/option.rs index 564a4084c..1ed20f5b5 100644 --- a/meilisearch/src/option.rs +++ b/meilisearch/src/option.rs @@ -30,7 +30,6 @@ const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY"; const MEILI_ENV: &str = "MEILI_ENV"; #[cfg(feature = "analytics")] const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS"; -const MEILI_MAX_NUMBER_OF_BATCHED_TASKS: &str = "MEILI_MAX_NUMBER_OF_BATCHED_TASKS"; const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT"; const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH"; const MEILI_SSL_KEY_PATH: &str = "MEILI_SSL_KEY_PATH"; @@ -52,6 +51,8 @@ const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL"; const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS"; const MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: &str = "MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE"; +const MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS: &str = + "MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS"; const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml"; const DEFAULT_DB_PATH: &str = "./data.ms"; @@ -176,11 +177,6 @@ pub struct Opt { #[serde(skip, default = "default_max_task_db_size")] pub max_task_db_size: Byte, - /// Defines the maximum number of tasks that will be processed at once. - #[clap(long, env = MEILI_MAX_NUMBER_OF_BATCHED_TASKS, default_value_t = default_limit_batched_tasks())] - #[serde(default = "default_limit_batched_tasks")] - pub max_number_of_batched_tasks: usize, - /// Sets the maximum size of accepted payloads. Value must be given in bytes or explicitly stating a /// base unit (for instance: 107374182400, '107.7Gb', or '107374 Mb'). #[clap(long, env = MEILI_HTTP_PAYLOAD_SIZE_LIMIT, default_value_t = default_http_payload_size_limit())] @@ -307,6 +303,11 @@ pub struct Opt { #[serde(default)] pub experimental_reduce_indexing_memory_usage: bool, + /// Experimentally reduces the maximum number of tasks that will be processed at once, see: + #[clap(long, env = MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS, default_value_t = default_limit_batched_tasks())] + #[serde(default = "default_limit_batched_tasks")] + pub experimental_max_number_of_batched_tasks: usize, + #[serde(flatten)] #[clap(flatten)] pub indexer_options: IndexerOpts, @@ -377,7 +378,7 @@ impl Opt { max_index_size: _, max_task_db_size: _, http_payload_size_limit, - max_number_of_batched_tasks, + experimental_max_number_of_batched_tasks, ssl_cert_path, ssl_key_path, ssl_auth_path, @@ -417,8 +418,8 @@ impl Opt { http_payload_size_limit.to_string(), ); export_to_env_if_not_present( - MEILI_MAX_NUMBER_OF_BATCHED_TASKS, - max_number_of_batched_tasks.to_string(), + MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS, + experimental_max_number_of_batched_tasks.to_string(), ); if let Some(ssl_cert_path) = ssl_cert_path { export_to_env_if_not_present(MEILI_SSL_CERT_PATH, ssl_cert_path);