From c0f9c891f512c1e622b7a929337b5af2e12ce4a0 Mon Sep 17 00:00:00 2001 From: many Date: Mon, 6 Sep 2021 13:46:19 +0200 Subject: [PATCH 1/3] Set max_memory value to unlimited during tests because tests run several meilisearch in parallel, we over estimate the value for max_memory making the tests on widows crash --- meilisearch-http/src/option.rs | 6 ++++++ meilisearch-http/tests/common/server.rs | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index f3c077c05..e5aa1553a 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -286,6 +286,12 @@ impl Deref for MaxMemory { } } +impl MaxMemory { + pub fn unlimited() -> Self { + Self(None) + } +} + /// Returns the total amount of bytes available or `None` if this system isn't supported. fn total_memory_bytes() -> Option { if System::IS_SUPPORTED { diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 6cf1acb6a..52c0e30ea 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -7,7 +7,7 @@ use tempdir::TempDir; use urlencoding::encode; use meilisearch_http::data::Data; -use meilisearch_http::option::{IndexerOpts, Opt}; +use meilisearch_http::option::{IndexerOpts, MaxMemory, Opt}; use super::index::Index; use super::service::Service; @@ -90,7 +90,11 @@ pub fn default_settings(dir: impl AsRef) -> Opt { schedule_snapshot: false, snapshot_interval_sec: 0, import_dump: None, - indexer_options: IndexerOpts::default(), + indexer_options: IndexerOpts { + // memory has to be unlimited because several meilisearch are running in test context. + max_memory: MaxMemory::unlimited(), + ..Default::default() + }, log_level: "off".into(), } } From 08138c7c236c153d8dfc8ef109f7d151900ba714 Mon Sep 17 00:00:00 2001 From: many Date: Wed, 8 Sep 2021 10:44:23 +0200 Subject: [PATCH 2/3] Use set indexer options instead of create a default one --- .../src/index_controller/index_actor/actor.rs | 9 ++++++--- .../src/index_controller/index_actor/handle_impl.rs | 9 +++++++-- meilisearch-http/src/index_controller/mod.rs | 3 ++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index 15d96b7ad..fc40a5090 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -31,9 +31,12 @@ pub struct IndexActor { } impl IndexActor { - pub fn new(receiver: mpsc::Receiver, store: S) -> anyhow::Result { - let options = IndexerOpts::default(); - let update_handler = UpdateHandler::new(&options)?; + pub fn new( + receiver: mpsc::Receiver, + store: S, + options: &IndexerOpts, + ) -> anyhow::Result { + let update_handler = UpdateHandler::new(options)?; let update_handler = Arc::new(update_handler); let receiver = Some(receiver); Ok(Self { diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 231a3a44b..ceb2a8226 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -1,3 +1,4 @@ +use crate::option::IndexerOpts; use std::path::{Path, PathBuf}; use tokio::sync::{mpsc, oneshot}; @@ -148,11 +149,15 @@ impl IndexActorHandle for IndexActorHandleImpl { } impl IndexActorHandleImpl { - pub fn new(path: impl AsRef, index_size: usize) -> anyhow::Result { + pub fn new( + path: impl AsRef, + index_size: usize, + options: &IndexerOpts, + ) -> anyhow::Result { let (sender, receiver) = mpsc::channel(100); let store = MapIndexStore::new(path, index_size); - let actor = IndexActor::new(receiver, store)?; + let actor = IndexActor::new(receiver, store, options)?; tokio::task::spawn(actor.run()); Ok(Self { sender }) } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index a90498b9c..83a97cd8f 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -110,7 +110,8 @@ impl IndexController { std::fs::create_dir_all(&path)?; let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&path)?; - let index_handle = index_actor::IndexActorHandleImpl::new(&path, index_size)?; + let index_handle = + index_actor::IndexActorHandleImpl::new(&path, index_size, &options.indexer_options)?; let update_handle = update_actor::UpdateActorHandleImpl::new( index_handle.clone(), &path, From 169e7396348563ecb670c8a3e8b8e1cad594b067 Mon Sep 17 00:00:00 2001 From: many Date: Wed, 8 Sep 2021 11:22:27 +0200 Subject: [PATCH 3/3] Remove useless indexer options --- meilisearch-http/src/option.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index e5aa1553a..39966092e 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -38,11 +38,6 @@ pub struct IndexerOpts { #[structopt(long, default_value)] pub max_memory: MaxMemory, - /// Size of the linked hash map cache when indexing. - /// The bigger it is, the faster the indexing is but the more memory it takes. - #[structopt(long, default_value = "500")] - pub linked_hash_map_size: usize, - /// The name of the compression algorithm to use when compressing intermediate /// Grenad chunks while indexing documents. /// @@ -54,18 +49,6 @@ pub struct IndexerOpts { #[structopt(long, requires = "chunk-compression-type")] pub chunk_compression_level: Option, - /// The number of bytes to remove from the begining of the chunks while reading/sorting - /// or merging them. - /// - /// File fusing must only be enable on file systems that support the `FALLOC_FL_COLLAPSE_RANGE`, - /// (i.e. ext4 and XFS). File fusing will only work if the `enable-chunk-fusing` is set. - #[structopt(long, default_value = "4 GiB")] - pub chunk_fusing_shrink_size: Byte, - - /// Enable the chunk fusing or not, this reduces the amount of disk space used. - #[structopt(long)] - pub enable_chunk_fusing: bool, - /// Number of parallel jobs for indexing, defaults to # of CPUs. #[structopt(long)] pub indexing_jobs: Option, @@ -77,11 +60,8 @@ impl Default for IndexerOpts { log_every_n: 100_000, max_nb_chunks: None, max_memory: MaxMemory::default(), - linked_hash_map_size: 500, chunk_compression_type: CompressionType::None, chunk_compression_level: None, - chunk_fusing_shrink_size: Byte::from_str("4GiB").unwrap(), - enable_chunk_fusing: false, indexing_jobs: None, } }