Merge branch 'fix-3037' of github.com:jiangbo212/meilisearch into fix-3037

.github/scripts/is-latest-release.sh

@@ -1,132 +0,0 @@
-#!/bin/sh
-
-# Was used in our CIs to publish the latest docker image. Not used anymore, will be used again when v1 and v2 will be out and we will want to maintain multiple stable versions.
-# Returns "true" or "false" (as a string) to be used in the `if` in GHA
-
-# Checks if the current tag should be the latest (in terms of semver and not of release date).
-# Ex: previous tag -> v2.1.1
-#     new tag -> v1.20.3
-#     The new tag (v1.20.3) should NOT be the latest
-#     So it returns "false", the `latest` tag should not be updated for the release v1.20.3 and still need to correspond to v2.1.1
-
-# GLOBAL
-GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
-
-# FUNCTIONS
-
-# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
-
-# usage: semverParseInto version major minor patch special
-# version: the string version
-# major, minor, patch, special: will be assigned by the function
-semverParseInto() {
-    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
-    #MAJOR
-    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
-    #MINOR
-    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
-    #MINOR
-    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
-    #SPECIAL
-    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
-}
-
-# usage: semverLT version1 version2
-semverLT() {
-    local MAJOR_A=0
-    local MINOR_A=0
-    local PATCH_A=0
-    local SPECIAL_A=0
-
-    local MAJOR_B=0
-    local MINOR_B=0
-    local PATCH_B=0
-    local SPECIAL_B=0
-
-    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
-    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
-
-    if [ $MAJOR_A -lt $MAJOR_B ]; then
-        return 0
-    fi
-    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
-        return 0
-    fi
-    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
-        return 0
-    fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
-        return 1
-    fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
-        return 1
-    fi
-    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
-        return 0
-    fi
-    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
-        return 0
-    fi
-
-    return 1
-}
-
-# Returns the tag of the latest stable release (in terms of semver and not of release date)
-get_latest() {
-    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
-    curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file"
-    releases=$(cat "$temp_file" | \
-        grep -E "tag_name|draft|prerelease" \
-        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
-        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
-        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
-
-    i=0
-    latest=""
-    current_tag=""
-    for release_info in $releases; do
-        if [ $i -eq 0 ]; then # Checking tag_name
-            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
-                current_tag=$release_info
-            else
-                current_tag=""
-            fi
-            i=1
-        elif [ $i -eq 1 ]; then # Checking draft boolean
-            if [ "$release_info" = "true" ]; then
-                current_tag=""
-            fi
-            i=2
-        elif [ $i -eq 2 ]; then # Checking prerelease boolean
-            if [ "$release_info" = "true" ]; then
-                current_tag=""
-            fi
-            i=0
-            if [ "$current_tag" != "" ]; then # If the current_tag is valid
-                if [ "$latest" = "" ]; then # If there is no latest yet
-                    latest="$current_tag"
-                else
-                    semverLT $current_tag $latest # Comparing latest and the current tag
-                    if [ $? -eq 1 ]; then
-                        latest="$current_tag"
-                    fi
-                fi
-            fi
-        fi
-    done
-
-    rm -f "$temp_file"
-    echo $latest
-}
-
-# MAIN
-current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')"
-latest="$(get_latest)"
-
-if [ "$current_tag" != "$latest" ]; then
-    # The current release tag is not the latest
-    echo "false"
-else
-    # The current release tag is the latest
-    echo "true"
-fi

.github/workflows/latest-git-tag.yml

@@ -1,4 +1,4 @@
-# Create or update a latest git tag when releasing a stable vesrsin of Meilisearch
+# Create or update a latest git tag when releasing a stable version of Meilisearch
 name: Update latest git tag
 on:
   workflow_dispatch:
@@ -6,6 +6,15 @@ on:
     types: [released]
 
 jobs:
+  check-version:
+    name: Check the version validity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check release validity
+        if: github.event_name == 'release'
+        run: bash .github/scripts/check-release.sh
+
   update-latest-tag:
     runs-on: ubuntu-latest
     steps:

Cargo.lock

@@ -617,9 +617,9 @@ dependencies = [
 
 [[package]]
 name = "cargo_toml"
-version = "0.12.4"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a621d5d6d6c8d086dbaf1fe659981da41a1b63c6bdbba30b4dbb592c6d3bd49"
+checksum = "aa0e3586af56b3bfa51fca452bd56e8dbbbd5d8d81cbf0b7e4e35b695b537eb8"
 dependencies = [
  "serde",
  "toml",
@@ -1101,7 +1101,7 @@ dependencies = [
 
 [[package]]
 name = "dump"
-version = "0.30.0"
+version = "0.30.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -1310,7 +1310,7 @@ dependencies = [
 
 [[package]]
 name = "file-store"
-version = "0.30.0"
+version = "0.30.1"
 dependencies = [
  "faux",
  "tempfile",
@@ -1332,8 +1332,8 @@ dependencies = [
 
 [[package]]
 name = "filter-parser"
-version = "0.37.0"
+version = "0.37.1"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0"
+source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
 dependencies = [
  "nom",
  "nom_locate",
@@ -1351,8 +1351,8 @@ dependencies = [
 
 [[package]]
 name = "flatten-serde-json"
-version = "0.37.0"
+version = "0.37.1"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0"
+source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
 dependencies = [
  "serde_json",
 ]
@@ -1625,7 +1625,7 @@ dependencies = [
  "libc",
  "lmdb-rkv-sys",
  "once_cell",
- "page_size",
+ "page_size 0.4.2",
  "synchronoise",
  "url",
  "zerocopy",
@@ -1767,7 +1767,7 @@ dependencies = [
 
 [[package]]
 name = "index-scheduler"
-version = "0.30.0"
+version = "0.30.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -1783,6 +1783,7 @@ dependencies = [
  "meili-snap",
  "meilisearch-types",
  "nelson",
+ "page_size 0.5.0",
  "roaring",
  "serde",
  "serde_json",
@@ -1897,8 +1898,8 @@ dependencies = [
 
 [[package]]
 name = "json-depth-checker"
-version = "0.37.0"
+version = "0.37.1"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0"
+source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
 dependencies = [
  "serde_json",
 ]
@@ -2257,7 +2258,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "meili-snap"
-version = "0.30.0"
+version = "0.30.1"
 dependencies = [
  "insta",
  "md5",
@@ -2266,7 +2267,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-auth"
-version = "0.30.0"
+version = "0.30.1"
 dependencies = [
  "enum-iterator",
  "hmac",
@@ -2283,7 +2284,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-http"
-version = "0.30.0"
+version = "0.30.1"
 dependencies = [
  "actix-cors",
  "actix-http",
@@ -2366,7 +2367,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-types"
-version = "0.30.0"
+version = "0.30.1"
 dependencies = [
  "actix-web",
  "anyhow",
@@ -2417,8 +2418,8 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "0.37.0"
+version = "0.37.1"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.0#57c9f03e514436a2cca799b2a28cd89247682be0"
+source = "git+https://github.com/meilisearch/milli.git?tag=v0.37.1#e1b2113cd9a467410841a254286e5e25961af43e"
 dependencies = [
  "bimap",
  "bincode",
@@ -2664,6 +2665,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "page_size"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -2748,7 +2759,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
 
 [[package]]
 name = "permissive-json-pointer"
-version = "0.30.0"
+version = "0.30.1"
 dependencies = [
  "big_s",
  "serde_json",

dump/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "dump"
-version = "0.30.0"
+version = "0.30.1"
 edition = "2021"
 
 [dependencies]

dump/src/writer.rs

@@ -292,10 +292,10 @@ pub(crate) mod test {
         // ==== checking the top level infos
         let metadata = fs::read_to_string(dump_path.join("metadata.json")).unwrap();
         let metadata: Metadata = serde_json::from_str(&metadata).unwrap();
-        insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]" }, @r###"
+        insta::assert_json_snapshot!(metadata, { ".dumpDate" => "[date]", ".dbVersion" => "[version]" }, @r###"
         {
           "dumpVersion": "V6",
-          "dbVersion": "0.30.0",
+          "dbVersion": "[version]",
           "dumpDate": "[date]"
         }
         "###);

file-store/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "file-store"
-version = "0.30.0"
+version = "0.30.1"
 edition = "2021"
 
 [dependencies]

index-scheduler/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "index-scheduler"
-version = "0.30.0"
+version = "0.30.1"
 edition = "2021"
 
 [dependencies]
@@ -13,6 +13,7 @@ enum-iterator = "1.1.3"
 file-store = { path = "../file-store" }
 log = "0.4.14"
 meilisearch-types = { path = "../meilisearch-types" }
+page_size = "0.5.0"
 roaring = { version = "0.10.0", features = ["serde"] }
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = { version = "1.0.85", features = ["preserve_order"] }

index-scheduler/src/index_mapper.rs

@@ -13,7 +13,7 @@ use uuid::Uuid;
 
 use self::IndexStatus::{Available, BeingDeleted};
 use crate::uuid_codec::UuidCodec;
-use crate::{Error, Result};
+use crate::{clamp_to_page_size, Error, Result};
 
 const INDEX_MAPPING: &str = "index-mapping";
 
@@ -68,7 +68,7 @@ impl IndexMapper {
     /// The path *must* exists or an error will be thrown.
     fn create_or_open_index(&self, path: &Path) -> Result<Index> {
         let mut options = EnvOpenOptions::new();
-        options.map_size(self.index_size);
+        options.map_size(clamp_to_page_size(self.index_size));
         options.max_readers(1024);
         Ok(Index::new(options, path)?)
     }

index-scheduler/src/lib.rs

@@ -55,7 +55,7 @@ use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, m
 use uuid::Uuid;
 
 use crate::index_mapper::IndexMapper;
-use crate::utils::check_index_swap_validity;
+use crate::utils::{check_index_swap_validity, clamp_to_page_size};
 
 pub(crate) type BEI128 =
     meilisearch_types::heed::zerocopy::I128<meilisearch_types::heed::byteorder::BE>;
@@ -362,7 +362,7 @@ impl IndexScheduler {
 
         let env = heed::EnvOpenOptions::new()
             .max_dbs(10)
-            .map_size(options.task_db_size)
+            .map_size(clamp_to_page_size(options.task_db_size))
             .open(options.tasks_path)?;
         let file_store = FileStore::new(&options.update_file_path)?;
 
@@ -1114,8 +1114,8 @@ mod tests {
                 indexes_path: tempdir.path().join("indexes"),
                 snapshots_path: tempdir.path().join("snapshots"),
                 dumps_path: tempdir.path().join("dumps"),
-                task_db_size: 1024 * 1024, // 1 MiB
+                task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
-                index_size: 1024 * 1024,   // 1 MiB
+                index_size: 1000 * 1000,   // 1 MB, we don't use MiB on purpose.
                 indexer_config: IndexerConfig::default(),
                 autobatching_enabled,
             };

index-scheduler/src/utils.rs

@@ -324,6 +324,11 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> {
     Ok(())
 }
 
+/// Clamp the provided value to be a multiple of system page size.
+pub fn clamp_to_page_size(size: usize) -> usize {
+    size / page_size::get() * page_size::get()
+}
+
 #[cfg(test)]
 impl IndexScheduler {
     /// Asserts that the index scheduler's content is internally consistent.

meili-snap/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "meili-snap"
-version = "0.30.0"
+version = "0.30.1"
 edition = "2021"
 
 [dependencies]

meilisearch-auth/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "meilisearch-auth"
-version = "0.30.0"
+version = "0.30.1"
 edition = "2021"
 
 [dependencies]

meilisearch-http/Cargo.toml

@@ -4,7 +4,7 @@ description = "Meilisearch HTTP server"
 edition = "2021"
 license = "MIT"
 name = "meilisearch-http"
-version = "0.30.0"
+version = "0.30.1"
 
 [[bin]]
 name = "meilisearch"
@@ -89,7 +89,7 @@ yaup = "0.2.1"
 
 [build-dependencies]
 anyhow = { version = "1.0.65", optional = true }
-cargo_toml = { version = "0.12.4", optional = true }
+cargo_toml = { version = "0.13.0", optional = true }
 hex = { version = "0.4.3", optional = true }
 reqwest = { version = "0.11.12", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
 sha-1 = { version = "0.10.0", optional = true }

meilisearch-types/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "meilisearch-types"
-version = "0.30.0"
+version = "0.30.1"
 authors = ["marin <postma.marin@protonmail.com>"]
 edition = "2021"
 
@@ -13,7 +13,7 @@ enum-iterator = "1.1.3"
 flate2 = "1.0.24"
 fst = "0.4.7"
 memmap2 = "0.5.7"
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.0", default-features = false }
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.37.1", default-features = false }
 proptest = { version = "1.0.0", optional = true }
 proptest-derive = { version = "0.3.0", optional = true }
 roaring = { version = "0.10.0", features = ["serde"] }

permissive-json-pointer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "permissive-json-pointer"
-version = "0.30.0"
+version = "0.30.1"
 edition = "2021"
 description = "A permissive json pointer"
 readme = "README.md"