From 5db7c4057c3c7c9c8be0bf966112b4d228dea3b2 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 6 Dec 2022 16:20:31 +0000 Subject: [PATCH 1/5] Update version for the next release (v0.30.1) in Cargo.toml files --- Cargo.lock | 16 ++++++++-------- dump/Cargo.toml | 2 +- file-store/Cargo.toml | 2 +- index-scheduler/Cargo.toml | 2 +- meili-snap/Cargo.toml | 2 +- meilisearch-auth/Cargo.toml | 2 +- meilisearch-http/Cargo.toml | 2 +- meilisearch-types/Cargo.toml | 2 +- permissive-json-pointer/Cargo.toml | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 525b92976..5918f15c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1101,7 +1101,7 @@ dependencies = [ [[package]] name = "dump" -version = "0.30.0" +version = "0.30.1" dependencies = [ "anyhow", "big_s", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "file-store" -version = "0.30.0" +version = "0.30.1" dependencies = [ "faux", "tempfile", @@ -1767,7 +1767,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "0.30.0" +version = "0.30.1" dependencies = [ "anyhow", "big_s", @@ -2257,7 +2257,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "0.30.0" +version = "0.30.1" dependencies = [ "insta", "md5", @@ -2266,7 +2266,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "0.30.0" +version = "0.30.1" dependencies = [ "enum-iterator", "hmac", @@ -2283,7 +2283,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.30.0" +version = "0.30.1" dependencies = [ "actix-cors", "actix-http", @@ -2366,7 +2366,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "0.30.0" +version = "0.30.1" dependencies = [ "actix-web", "anyhow", @@ -2747,7 +2747,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "permissive-json-pointer" -version = "0.30.0" +version = "0.30.1" dependencies = [ "big_s", "serde_json", diff --git a/dump/Cargo.toml b/dump/Cargo.toml index c5dc10949..f6850f71c 100644 --- a/dump/Cargo.toml +++ b/dump/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dump" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/file-store/Cargo.toml b/file-store/Cargo.toml index 0110a00b1..3adef3315 100644 --- a/file-store/Cargo.toml +++ b/file-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-store" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index e46712703..b4284abea 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "index-scheduler" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/meili-snap/Cargo.toml b/meili-snap/Cargo.toml index 6706bf693..1682ddc11 100644 --- a/meili-snap/Cargo.toml +++ b/meili-snap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meili-snap" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index f0b73b539..1df43411e 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-auth" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 1d451f9d0..2cf101d97 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "Meilisearch HTTP server" edition = "2021" license = "MIT" name = "meilisearch-http" -version = "0.30.0" +version = "0.30.1" [[bin]] name = "meilisearch" diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 81aeaaa69..32e2e1971 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-types" -version = "0.30.0" +version = "0.30.1" authors = ["marin "] edition = "2021" diff --git a/permissive-json-pointer/Cargo.toml b/permissive-json-pointer/Cargo.toml index b036a0c88..3343ee41f 100644 --- a/permissive-json-pointer/Cargo.toml +++ b/permissive-json-pointer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "permissive-json-pointer" -version = "0.30.0" +version = "0.30.1" edition = "2021" description = "A permissive json pointer" readme = "README.md" From 24c79b79f9bfe0a35a17088e2d891ff5e44db7b6 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 6 Dec 2022 17:11:03 +0100 Subject: [PATCH 2/5] Bump milli to v0.37.1 --- Cargo.lock | 16 ++++++++-------- meilisearch-types/Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5918f15c4..26fcc72a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1332,8 +1332,8 @@ dependencies = [ [[package]] name = "filter-parser" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "nom", "nom_locate", @@ -1351,8 +1351,8 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "serde_json", ] @@ -1897,8 +1897,8 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "serde_json", ] @@ -2416,8 +2416,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "bimap", "bincode", diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 32e2e1971..a36284adf 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -12,7 +12,7 @@ either = { version = "1.6.1", features = ["serde"] } enum-iterator = "1.1.3" flate2 = "1.0.24" fst = "0.4.7" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.0", default-features = false } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.1", default-features = false } proptest = { version = "1.0.0", optional = true } proptest-derive = { version = "0.3.0", optional = true } roaring = { version = "0.10.0", features = ["serde"] } From e4e4370a3c357ddc93dc3a10f84404f50341baad Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 29 Nov 2022 10:38:27 +0100 Subject: [PATCH 3/5] Clamp the databases size to the page size --- Cargo.lock | 13 ++++++++++++- index-scheduler/Cargo.toml | 1 + index-scheduler/src/index_mapper.rs | 4 ++-- index-scheduler/src/lib.rs | 4 ++-- index-scheduler/src/utils.rs | 5 +++++ 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26fcc72a7..b1a2b954c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1625,7 +1625,7 @@ dependencies = [ "libc", "lmdb-rkv-sys", "once_cell", - "page_size", + "page_size 0.4.2", "synchronoise", "url", "zerocopy", @@ -1783,6 +1783,7 @@ dependencies = [ "meili-snap", "meilisearch-types", "nelson", + "page_size 0.5.0", "roaring", "serde", "serde_json", @@ -2663,6 +2664,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "page_size" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.1" diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index b4284abea..fa324d98b 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -13,6 +13,7 @@ enum-iterator = "1.1.3" file-store = { path = "../file-store" } log = "0.4.14" meilisearch-types = { path = "../meilisearch-types" } +page_size = "0.5.0" roaring = { version = "0.10.0", features = ["serde"] } serde = { version = "1.0.136", features = ["derive"] } serde_json = { version = "1.0.85", features = ["preserve_order"] } diff --git a/index-scheduler/src/index_mapper.rs b/index-scheduler/src/index_mapper.rs index 8869591f7..c65d76837 100644 --- a/index-scheduler/src/index_mapper.rs +++ b/index-scheduler/src/index_mapper.rs @@ -13,7 +13,7 @@ use uuid::Uuid; use self::IndexStatus::{Available, BeingDeleted}; use crate::uuid_codec::UuidCodec; -use crate::{Error, Result}; +use crate::{clamp_to_page_size, Error, Result}; const INDEX_MAPPING: &str = "index-mapping"; @@ -68,7 +68,7 @@ impl IndexMapper { /// The path *must* exists or an error will be thrown. fn create_or_open_index(&self, path: &Path) -> Result { let mut options = EnvOpenOptions::new(); - options.map_size(self.index_size); + options.map_size(clamp_to_page_size(self.index_size)); options.max_readers(1024); Ok(Index::new(options, path)?) } diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 9e32a40b8..af4f637c7 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -55,7 +55,7 @@ use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, m use uuid::Uuid; use crate::index_mapper::IndexMapper; -use crate::utils::check_index_swap_validity; +use crate::utils::{check_index_swap_validity, clamp_to_page_size}; pub(crate) type BEI128 = meilisearch_types::heed::zerocopy::I128; @@ -362,7 +362,7 @@ impl IndexScheduler { let env = heed::EnvOpenOptions::new() .max_dbs(10) - .map_size(options.task_db_size) + .map_size(clamp_to_page_size(options.task_db_size)) .open(options.tasks_path)?; let file_store = FileStore::new(&options.update_file_path)?; diff --git a/index-scheduler/src/utils.rs b/index-scheduler/src/utils.rs index a193c2bec..0018ae1d0 100644 --- a/index-scheduler/src/utils.rs +++ b/index-scheduler/src/utils.rs @@ -324,6 +324,11 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> { Ok(()) } +/// Clamp the provided value to be a multiple of system page size. +pub fn clamp_to_page_size(size: usize) -> usize { + size / page_size::get() * page_size::get() +} + #[cfg(test)] impl IndexScheduler { /// Asserts that the index scheduler's content is internally consistent. From f1de3aa75a7269472c5b1b5caa6fca478be54517 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 29 Nov 2022 15:26:24 +0100 Subject: [PATCH 4/5] Make the tests use MB to trigger page size issues --- index-scheduler/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index af4f637c7..b57adb077 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -1112,8 +1112,8 @@ mod tests { indexes_path: tempdir.path().join("indexes"), snapshots_path: tempdir.path().join("snapshots"), dumps_path: tempdir.path().join("dumps"), - task_db_size: 1024 * 1024, // 1 MiB - index_size: 1024 * 1024, // 1 MiB + task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. + index_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. indexer_config: IndexerConfig::default(), autobatching_enabled, }; From c1337f9e08f6fab8946aa44ee7bbdd716ea1ac67 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 7 Dec 2022 11:48:29 +0100 Subject: [PATCH 5/5] Update dump snap to new version --- dump/src/writer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/src/writer.rs b/dump/src/writer.rs index c11cf6b9a..1e61fc295 100644 --- a/dump/src/writer.rs +++ b/dump/src/writer.rs @@ -295,7 +295,7 @@ pub(crate) mod test { insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]" }, @r###" { "dumpVersion": "V6", - "dbVersion": "0.30.0", + "dbVersion": "0.30.1", "dumpDate": "[date]" } "###);