diff --git a/Cargo.lock b/Cargo.lock index 525b92976..791c7676d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1625,7 +1625,7 @@ dependencies = [ "libc", "lmdb-rkv-sys", "once_cell", - "page_size", + "page_size 0.4.2", "synchronoise", "url", "zerocopy", @@ -1783,6 +1783,7 @@ dependencies = [ "meili-snap", "meilisearch-types", "nelson", + "page_size 0.5.0", "roaring", "serde", "serde_json", @@ -2663,6 +2664,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "page_size" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.1" diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index e46712703..1ebb1ce36 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -13,6 +13,7 @@ enum-iterator = "1.1.3" file-store = { path = "../file-store" } log = "0.4.14" meilisearch-types = { path = "../meilisearch-types" } +page_size = "0.5.0" roaring = { version = "0.10.0", features = ["serde"] } serde = { version = "1.0.136", features = ["derive"] } serde_json = { version = "1.0.85", features = ["preserve_order"] } diff --git a/index-scheduler/src/index_mapper.rs b/index-scheduler/src/index_mapper.rs index a647012fe..064a625c8 100644 --- a/index-scheduler/src/index_mapper.rs +++ b/index-scheduler/src/index_mapper.rs @@ -12,7 +12,7 @@ use meilisearch_types::milli::Index; use uuid::Uuid; use self::IndexStatus::{Available, BeingDeleted}; -use crate::{Error, Result}; +use crate::{clamp_to_page_size, Error, Result}; const INDEX_MAPPING: &str = "index-mapping"; @@ -68,7 +68,7 @@ impl IndexMapper { /// The path *must* exists or an error will be thrown. fn create_or_open_index(&self, path: &Path) -> Result { let mut options = EnvOpenOptions::new(); - options.map_size(self.index_size); + options.map_size(clamp_to_page_size(self.index_size)); options.max_readers(1024); Ok(Index::new(options, path)?) } diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index d61913c09..b0ab1e49d 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -54,7 +54,7 @@ use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, m use uuid::Uuid; use crate::index_mapper::IndexMapper; -use crate::utils::check_index_swap_validity; +use crate::utils::{check_index_swap_validity, clamp_to_page_size}; pub(crate) type BEI128 = meilisearch_types::heed::zerocopy::I128; @@ -361,7 +361,7 @@ impl IndexScheduler { let env = heed::EnvOpenOptions::new() .max_dbs(10) - .map_size(options.task_db_size) + .map_size(clamp_to_page_size(options.task_db_size)) .open(options.tasks_path)?; let file_store = FileStore::new(&options.update_file_path)?; diff --git a/index-scheduler/src/utils.rs b/index-scheduler/src/utils.rs index a193c2bec..0018ae1d0 100644 --- a/index-scheduler/src/utils.rs +++ b/index-scheduler/src/utils.rs @@ -324,6 +324,11 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> { Ok(()) } +/// Clamp the provided value to be a multiple of system page size. +pub fn clamp_to_page_size(size: usize) -> usize { + size / page_size::get() * page_size::get() +} + #[cfg(test)] impl IndexScheduler { /// Asserts that the index scheduler's content is internally consistent.