From 6ecd486d1b765993882d9a1ed061ae50b2d1c162 Mon Sep 17 00:00:00 2001 From: Colby Allen Date: Wed, 23 Nov 2022 16:27:54 -0700 Subject: [PATCH 1/8] Bumps cargo_toml version to most up to date --- Cargo.lock | 4 ++-- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 01eb9cd36..63364f2d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -617,9 +617,9 @@ dependencies = [ [[package]] name = "cargo_toml" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a621d5d6d6c8d086dbaf1fe659981da41a1b63c6bdbba30b4dbb592c6d3bd49" +checksum = "aa0e3586af56b3bfa51fca452bd56e8dbbbd5d8d81cbf0b7e4e35b695b537eb8" dependencies = [ "serde", "toml", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 6a4cc0336..7dd722613 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -89,7 +89,7 @@ yaup = "0.2.1" [build-dependencies] anyhow = { version = "1.0.65", optional = true } -cargo_toml = { version = "0.12.4", optional = true } +cargo_toml = { version = "0.13.0", optional = true } hex = { version = "0.4.3", optional = true } reqwest = { version = "0.11.12", features = ["blocking", "rustls-tls"], default-features = false, optional = true } sha-1 = { version = "0.10.0", optional = true } From 5db7c4057c3c7c9c8be0bf966112b4d228dea3b2 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 6 Dec 2022 16:20:31 +0000 Subject: [PATCH 2/8] Update version for the next release (v0.30.1) in Cargo.toml files --- Cargo.lock | 16 ++++++++-------- dump/Cargo.toml | 2 +- file-store/Cargo.toml | 2 +- index-scheduler/Cargo.toml | 2 +- meili-snap/Cargo.toml | 2 +- meilisearch-auth/Cargo.toml | 2 +- meilisearch-http/Cargo.toml | 2 +- meilisearch-types/Cargo.toml | 2 +- permissive-json-pointer/Cargo.toml | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 525b92976..5918f15c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1101,7 +1101,7 @@ dependencies = [ [[package]] name = "dump" -version = "0.30.0" +version = "0.30.1" dependencies = [ "anyhow", "big_s", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "file-store" -version = "0.30.0" +version = "0.30.1" dependencies = [ "faux", "tempfile", @@ -1767,7 +1767,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "0.30.0" +version = "0.30.1" dependencies = [ "anyhow", "big_s", @@ -2257,7 +2257,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "0.30.0" +version = "0.30.1" dependencies = [ "insta", "md5", @@ -2266,7 +2266,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "0.30.0" +version = "0.30.1" dependencies = [ "enum-iterator", "hmac", @@ -2283,7 +2283,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.30.0" +version = "0.30.1" dependencies = [ "actix-cors", "actix-http", @@ -2366,7 +2366,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "0.30.0" +version = "0.30.1" dependencies = [ "actix-web", "anyhow", @@ -2747,7 +2747,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "permissive-json-pointer" -version = "0.30.0" +version = "0.30.1" dependencies = [ "big_s", "serde_json", diff --git a/dump/Cargo.toml b/dump/Cargo.toml index c5dc10949..f6850f71c 100644 --- a/dump/Cargo.toml +++ b/dump/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dump" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/file-store/Cargo.toml b/file-store/Cargo.toml index 0110a00b1..3adef3315 100644 --- a/file-store/Cargo.toml +++ b/file-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "file-store" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index e46712703..b4284abea 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "index-scheduler" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/meili-snap/Cargo.toml b/meili-snap/Cargo.toml index 6706bf693..1682ddc11 100644 --- a/meili-snap/Cargo.toml +++ b/meili-snap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meili-snap" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index f0b73b539..1df43411e 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-auth" -version = "0.30.0" +version = "0.30.1" edition = "2021" [dependencies] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 1d451f9d0..2cf101d97 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "Meilisearch HTTP server" edition = "2021" license = "MIT" name = "meilisearch-http" -version = "0.30.0" +version = "0.30.1" [[bin]] name = "meilisearch" diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 81aeaaa69..32e2e1971 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-types" -version = "0.30.0" +version = "0.30.1" authors = ["marin "] edition = "2021" diff --git a/permissive-json-pointer/Cargo.toml b/permissive-json-pointer/Cargo.toml index b036a0c88..3343ee41f 100644 --- a/permissive-json-pointer/Cargo.toml +++ b/permissive-json-pointer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "permissive-json-pointer" -version = "0.30.0" +version = "0.30.1" edition = "2021" description = "A permissive json pointer" readme = "README.md" From 24c79b79f9bfe0a35a17088e2d891ff5e44db7b6 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 6 Dec 2022 17:11:03 +0100 Subject: [PATCH 3/8] Bump milli to v0.37.1 --- Cargo.lock | 16 ++++++++-------- meilisearch-types/Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5918f15c4..26fcc72a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1332,8 +1332,8 @@ dependencies = [ [[package]] name = "filter-parser" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "nom", "nom_locate", @@ -1351,8 +1351,8 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "serde_json", ] @@ -1897,8 +1897,8 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "serde_json", ] @@ -2416,8 +2416,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.37.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0" +version = "0.37.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e" dependencies = [ "bimap", "bincode", diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 32e2e1971..a36284adf 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -12,7 +12,7 @@ either = { version = "1.6.1", features = ["serde"] } enum-iterator = "1.1.3" flate2 = "1.0.24" fst = "0.4.7" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.0", default-features = false } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.1", default-features = false } proptest = { version = "1.0.0", optional = true } proptest-derive = { version = "0.3.0", optional = true } roaring = { version = "0.10.0", features = ["serde"] } From e4e4370a3c357ddc93dc3a10f84404f50341baad Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 29 Nov 2022 10:38:27 +0100 Subject: [PATCH 4/8] Clamp the databases size to the page size --- Cargo.lock | 13 ++++++++++++- index-scheduler/Cargo.toml | 1 + index-scheduler/src/index_mapper.rs | 4 ++-- index-scheduler/src/lib.rs | 4 ++-- index-scheduler/src/utils.rs | 5 +++++ 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26fcc72a7..b1a2b954c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1625,7 +1625,7 @@ dependencies = [ "libc", "lmdb-rkv-sys", "once_cell", - "page_size", + "page_size 0.4.2", "synchronoise", "url", "zerocopy", @@ -1783,6 +1783,7 @@ dependencies = [ "meili-snap", "meilisearch-types", "nelson", + "page_size 0.5.0", "roaring", "serde", "serde_json", @@ -2663,6 +2664,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "page_size" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.1" diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index b4284abea..fa324d98b 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -13,6 +13,7 @@ enum-iterator = "1.1.3" file-store = { path = "../file-store" } log = "0.4.14" meilisearch-types = { path = "../meilisearch-types" } +page_size = "0.5.0" roaring = { version = "0.10.0", features = ["serde"] } serde = { version = "1.0.136", features = ["derive"] } serde_json = { version = "1.0.85", features = ["preserve_order"] } diff --git a/index-scheduler/src/index_mapper.rs b/index-scheduler/src/index_mapper.rs index 8869591f7..c65d76837 100644 --- a/index-scheduler/src/index_mapper.rs +++ b/index-scheduler/src/index_mapper.rs @@ -13,7 +13,7 @@ use uuid::Uuid; use self::IndexStatus::{Available, BeingDeleted}; use crate::uuid_codec::UuidCodec; -use crate::{Error, Result}; +use crate::{clamp_to_page_size, Error, Result}; const INDEX_MAPPING: &str = "index-mapping"; @@ -68,7 +68,7 @@ impl IndexMapper { /// The path *must* exists or an error will be thrown. fn create_or_open_index(&self, path: &Path) -> Result { let mut options = EnvOpenOptions::new(); - options.map_size(self.index_size); + options.map_size(clamp_to_page_size(self.index_size)); options.max_readers(1024); Ok(Index::new(options, path)?) } diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 9e32a40b8..af4f637c7 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -55,7 +55,7 @@ use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, m use uuid::Uuid; use crate::index_mapper::IndexMapper; -use crate::utils::check_index_swap_validity; +use crate::utils::{check_index_swap_validity, clamp_to_page_size}; pub(crate) type BEI128 = meilisearch_types::heed::zerocopy::I128; @@ -362,7 +362,7 @@ impl IndexScheduler { let env = heed::EnvOpenOptions::new() .max_dbs(10) - .map_size(options.task_db_size) + .map_size(clamp_to_page_size(options.task_db_size)) .open(options.tasks_path)?; let file_store = FileStore::new(&options.update_file_path)?; diff --git a/index-scheduler/src/utils.rs b/index-scheduler/src/utils.rs index a193c2bec..0018ae1d0 100644 --- a/index-scheduler/src/utils.rs +++ b/index-scheduler/src/utils.rs @@ -324,6 +324,11 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> { Ok(()) } +/// Clamp the provided value to be a multiple of system page size. +pub fn clamp_to_page_size(size: usize) -> usize { + size / page_size::get() * page_size::get() +} + #[cfg(test)] impl IndexScheduler { /// Asserts that the index scheduler's content is internally consistent. From f1de3aa75a7269472c5b1b5caa6fca478be54517 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 29 Nov 2022 15:26:24 +0100 Subject: [PATCH 5/8] Make the tests use MB to trigger page size issues --- index-scheduler/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index af4f637c7..b57adb077 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -1112,8 +1112,8 @@ mod tests { indexes_path: tempdir.path().join("indexes"), snapshots_path: tempdir.path().join("snapshots"), dumps_path: tempdir.path().join("dumps"), - task_db_size: 1024 * 1024, // 1 MiB - index_size: 1024 * 1024, // 1 MiB + task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. + index_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. indexer_config: IndexerConfig::default(), autobatching_enabled, }; From c1337f9e08f6fab8946aa44ee7bbdd716ea1ac67 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 7 Dec 2022 11:48:29 +0100 Subject: [PATCH 6/8] Update dump snap to new version --- dump/src/writer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/src/writer.rs b/dump/src/writer.rs index c11cf6b9a..1e61fc295 100644 --- a/dump/src/writer.rs +++ b/dump/src/writer.rs @@ -295,7 +295,7 @@ pub(crate) mod test { insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]" }, @r###" { "dumpVersion": "V6", - "dbVersion": "0.30.0", + "dbVersion": "0.30.1", "dumpDate": "[date]" } "###); From be300138e41d8f2cfed86c3eb022103e9d30faab Mon Sep 17 00:00:00 2001 From: curquiza Date: Wed, 7 Dec 2022 12:17:48 +0100 Subject: [PATCH 7/8] Add release check when starting latest CI --- .github/scripts/is-latest-release.sh | 132 --------------------------- .github/workflows/latest-git-tag.yml | 11 ++- 2 files changed, 10 insertions(+), 133 deletions(-) delete mode 100644 .github/scripts/is-latest-release.sh diff --git a/.github/scripts/is-latest-release.sh b/.github/scripts/is-latest-release.sh deleted file mode 100644 index 54f0a9d3a..000000000 --- a/.github/scripts/is-latest-release.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/sh - -# Was used in our CIs to publish the latest docker image. Not used anymore, will be used again when v1 and v2 will be out and we will want to maintain multiple stable versions. -# Returns "true" or "false" (as a string) to be used in the `if` in GHA - -# Checks if the current tag should be the latest (in terms of semver and not of release date). -# Ex: previous tag -> v2.1.1 -# new tag -> v1.20.3 -# The new tag (v1.20.3) should NOT be the latest -# So it returns "false", the `latest` tag should not be updated for the release v1.20.3 and still need to correspond to v2.1.1 - -# GLOBAL -GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number] - -# FUNCTIONS - -# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh - -# usage: semverParseInto version major minor patch special -# version: the string version -# major, minor, patch, special: will be assigned by the function -semverParseInto() { - local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' - #MAJOR - eval $2=`echo $1 | sed -e "s#$RE#\1#"` - #MINOR - eval $3=`echo $1 | sed -e "s#$RE#\2#"` - #MINOR - eval $4=`echo $1 | sed -e "s#$RE#\3#"` - #SPECIAL - eval $5=`echo $1 | sed -e "s#$RE#\4#"` -} - -# usage: semverLT version1 version2 -semverLT() { - local MAJOR_A=0 - local MINOR_A=0 - local PATCH_A=0 - local SPECIAL_A=0 - - local MAJOR_B=0 - local MINOR_B=0 - local PATCH_B=0 - local SPECIAL_B=0 - - semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A - semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B - - if [ $MAJOR_A -lt $MAJOR_B ]; then - return 0 - fi - if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then - return 0 - fi - if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then - return 0 - fi - if [ "_$SPECIAL_A" == "_" ] && [ "_$SPECIAL_B" == "_" ] ; then - return 1 - fi - if [ "_$SPECIAL_A" == "_" ] && [ "_$SPECIAL_B" != "_" ] ; then - return 1 - fi - if [ "_$SPECIAL_A" != "_" ] && [ "_$SPECIAL_B" == "_" ] ; then - return 0 - fi - if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then - return 0 - fi - - return 1 -} - -# Returns the tag of the latest stable release (in terms of semver and not of release date) -get_latest() { - temp_file='temp_file' # temp_file needed because the grep would start before the download is over - curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file" - releases=$(cat "$temp_file" | \ - grep -E "tag_name|draft|prerelease" \ - | tr -d ',"' | cut -d ':' -f2 | tr -d ' ') - # Returns a list of [tag_name draft_boolean prerelease_boolean ...] - # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false... - - i=0 - latest="" - current_tag="" - for release_info in $releases; do - if [ $i -eq 0 ]; then # Checking tag_name - if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release - current_tag=$release_info - else - current_tag="" - fi - i=1 - elif [ $i -eq 1 ]; then # Checking draft boolean - if [ "$release_info" = "true" ]; then - current_tag="" - fi - i=2 - elif [ $i -eq 2 ]; then # Checking prerelease boolean - if [ "$release_info" = "true" ]; then - current_tag="" - fi - i=0 - if [ "$current_tag" != "" ]; then # If the current_tag is valid - if [ "$latest" = "" ]; then # If there is no latest yet - latest="$current_tag" - else - semverLT $current_tag $latest # Comparing latest and the current tag - if [ $? -eq 1 ]; then - latest="$current_tag" - fi - fi - fi - fi - done - - rm -f "$temp_file" - echo $latest -} - -# MAIN -current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')" -latest="$(get_latest)" - -if [ "$current_tag" != "$latest" ]; then - # The current release tag is not the latest - echo "false" -else - # The current release tag is the latest - echo "true" -fi diff --git a/.github/workflows/latest-git-tag.yml b/.github/workflows/latest-git-tag.yml index 9d9339ca8..e8cbc50f3 100644 --- a/.github/workflows/latest-git-tag.yml +++ b/.github/workflows/latest-git-tag.yml @@ -1,4 +1,4 @@ -# Create or update a latest git tag when releasing a stable vesrsin of Meilisearch +# Create or update a latest git tag when releasing a stable version of Meilisearch name: Update latest git tag on: workflow_dispatch: @@ -6,6 +6,15 @@ on: types: [released] jobs: + check-version: + name: Check the version validity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Check release validity + if: github.event_name == 'release' + run: bash .github/scripts/check-release.sh + update-latest-tag: runs-on: ubuntu-latest steps: From 1c3a326199449bd682b20a4e30af0ae8e5fb312b Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 7 Dec 2022 13:26:02 +0100 Subject: [PATCH 8/8] stop snapshotting the version of meilisearch in the dump It might change and we don't want to update this test everytime we make a new release. --- dump/src/writer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dump/src/writer.rs b/dump/src/writer.rs index 1e61fc295..81695f55c 100644 --- a/dump/src/writer.rs +++ b/dump/src/writer.rs @@ -292,10 +292,10 @@ pub(crate) mod test { // ==== checking the top level infos let metadata = fs::read_to_string(dump_path.join("metadata.json")).unwrap(); let metadata: Metadata = serde_json::from_str(&metadata).unwrap(); - insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]" }, @r###" + insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]", ".dbVersion" => "[version]" }, @r###" { "dumpVersion": "V6", - "dbVersion": "0.30.1", + "dbVersion": "[version]", "dumpDate": "[date]" } "###);