snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 18:17:39 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Make the --max-number-of-batched-tasks argument experimental

Browse Source
This commit is contained in:
Clément Renault 2023-12-12 10:55:33 +01:00
parent 7e259cb0d2
commit 99fec27788
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
3 changed files with 14 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
meilisearch/src/analytics/segment_analytics.rs
View File

@ -251,6 +251,7 @@ struct Infos {
env: String, env: String,
experimental_enable_metrics: bool, experimental_enable_metrics: bool,
experimental_reduce_indexing_memory_usage: bool, experimental_reduce_indexing_memory_usage: bool,
experimental_max_number_of_batched_tasks: usize,
db_path: bool, db_path: bool,
import_dump: bool, import_dump: bool,
dump_dir: bool, dump_dir: bool,
@ -263,7 +264,6 @@ struct Infos {
ignore_snapshot_if_db_exists: bool, ignore_snapshot_if_db_exists: bool,
http_addr: bool, http_addr: bool,
http_payload_size_limit: Byte, http_payload_size_limit: Byte,
max_number_of_batched_tasks: usize,
log_level: String, log_level: String,
max_indexing_memory: MaxMemory, max_indexing_memory: MaxMemory,
max_indexing_threads: MaxThreads, max_indexing_threads: MaxThreads,
@ -286,13 +286,13 @@ impl From<Opt> for Infos {
db_path, db_path,
experimental_enable_metrics, experimental_enable_metrics,
experimental_reduce_indexing_memory_usage, experimental_reduce_indexing_memory_usage,
experimental_max_number_of_batched_tasks,
http_addr, http_addr,
master_key: _, master_key: _,
env, env,
max_index_size: _, max_index_size: _,
max_task_db_size: _, max_task_db_size: _,
http_payload_size_limit, http_payload_size_limit,
max_number_of_batched_tasks,
ssl_cert_path, ssl_cert_path,
ssl_key_path, ssl_key_path,
ssl_auth_path, ssl_auth_path,
@ -342,7 +342,7 @@ impl From<Opt> for Infos {
ignore_snapshot_if_db_exists, ignore_snapshot_if_db_exists,
http_addr: http_addr != default_http_addr(), http_addr: http_addr != default_http_addr(),
http_payload_size_limit, http_payload_size_limit,
max_number_of_batched_tasks, experimental_max_number_of_batched_tasks,
log_level: log_level.to_string(), log_level: log_level.to_string(),
max_indexing_memory, max_indexing_memory,
max_indexing_threads, max_indexing_threads,

2
meilisearch/src/lib.rs
View File

@ -234,7 +234,7 @@ fn open_or_create_database_unchecked(
indexer_config: (&opt.indexer_options).try_into()?, indexer_config: (&opt.indexer_options).try_into()?,
autobatching_enabled: true, autobatching_enabled: true,
max_number_of_tasks: 1_000_000, max_number_of_tasks: 1_000_000,
max_number_of_batched_tasks: opt.max_number_of_batched_tasks, max_number_of_batched_tasks: opt.experimental_max_number_of_batched_tasks,
index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize, index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize,
index_count: DEFAULT_INDEX_COUNT, index_count: DEFAULT_INDEX_COUNT,
instance_features, instance_features,

19
meilisearch/src/option.rs
View File

@ -30,7 +30,6 @@ const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY";
const MEILI_ENV: &str = "MEILI_ENV"; const MEILI_ENV: &str = "MEILI_ENV";
#[cfg(feature = "analytics")] #[cfg(feature = "analytics")]
const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS"; const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS";
const MEILI_MAX_NUMBER_OF_BATCHED_TASKS: &str = "MEILI_MAX_NUMBER_OF_BATCHED_TASKS";
const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT"; const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT";
const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH"; const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH";
const MEILI_SSL_KEY_PATH: &str = "MEILI_SSL_KEY_PATH"; const MEILI_SSL_KEY_PATH: &str = "MEILI_SSL_KEY_PATH";
@ -52,6 +51,8 @@ const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL";
const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS"; const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS";
const MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: &str = const MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: &str =
"MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE"; "MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE";
const MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS: &str =
"MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS";
const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml"; const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml";
const DEFAULT_DB_PATH: &str = "./data.ms"; const DEFAULT_DB_PATH: &str = "./data.ms";
@ -176,11 +177,6 @@ pub struct Opt {
#[serde(skip, default = "default_max_task_db_size")] #[serde(skip, default = "default_max_task_db_size")]
pub max_task_db_size: Byte, pub max_task_db_size: Byte,
/// Defines the maximum number of tasks that will be processed at once.
#[clap(long, env = MEILI_MAX_NUMBER_OF_BATCHED_TASKS, default_value_t = default_limit_batched_tasks())]
#[serde(default = "default_limit_batched_tasks")]
pub max_number_of_batched_tasks: usize,
/// Sets the maximum size of accepted payloads. Value must be given in bytes or explicitly stating a /// Sets the maximum size of accepted payloads. Value must be given in bytes or explicitly stating a
/// base unit (for instance: 107374182400, '107.7Gb', or '107374 Mb'). /// base unit (for instance: 107374182400, '107.7Gb', or '107374 Mb').
#[clap(long, env = MEILI_HTTP_PAYLOAD_SIZE_LIMIT, default_value_t = default_http_payload_size_limit())] #[clap(long, env = MEILI_HTTP_PAYLOAD_SIZE_LIMIT, default_value_t = default_http_payload_size_limit())]
@ -307,6 +303,11 @@ pub struct Opt {
#[serde(default)] #[serde(default)]
pub experimental_reduce_indexing_memory_usage: bool, pub experimental_reduce_indexing_memory_usage: bool,
/// Experimentally reduces the maximum number of tasks that will be processed at once, see: <https://github.com/orgs/meilisearch/discussions/713>
#[clap(long, env = MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS, default_value_t = default_limit_batched_tasks())]
#[serde(default = "default_limit_batched_tasks")]
pub experimental_max_number_of_batched_tasks: usize,
#[serde(flatten)] #[serde(flatten)]
#[clap(flatten)] #[clap(flatten)]
pub indexer_options: IndexerOpts, pub indexer_options: IndexerOpts,
@ -377,7 +378,7 @@ impl Opt {
max_index_size: _, max_index_size: _,
max_task_db_size: _, max_task_db_size: _,
http_payload_size_limit, http_payload_size_limit,
max_number_of_batched_tasks, experimental_max_number_of_batched_tasks,
ssl_cert_path, ssl_cert_path,
ssl_key_path, ssl_key_path,
ssl_auth_path, ssl_auth_path,
@ -417,8 +418,8 @@ impl Opt {
http_payload_size_limit.to_string(), http_payload_size_limit.to_string(),
); );
export_to_env_if_not_present( export_to_env_if_not_present(
MEILI_MAX_NUMBER_OF_BATCHED_TASKS, MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS,
max_number_of_batched_tasks.to_string(), experimental_max_number_of_batched_tasks.to_string(),
); );
if let Some(ssl_cert_path) = ssl_cert_path { if let Some(ssl_cert_path) = ssl_cert_path {
export_to_env_if_not_present(MEILI_SSL_CERT_PATH, ssl_cert_path); export_to_env_if_not_present(MEILI_SSL_CERT_PATH, ssl_cert_path);