From 0cfad7eeacca550df4852076b47f7c988a731217 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 7 Jun 2022 16:23:23 +0200 Subject: [PATCH 01/23] bump milli version --- Cargo.lock | 16 ++++++++-------- meilisearch-auth/Cargo.toml | 2 +- meilisearch-lib/Cargo.toml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 193d8f75c..e2b36dbee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1092,8 +1092,8 @@ dependencies = [ [[package]] name = "filter-parser" -version = "0.26.4" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.5#1f6dc31e2f8ee02cdda255a856d15f253daf17ec" +version = "0.26.6" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.6#955aff48853c6032c75970dc710929f08b4f711a" dependencies = [ "nom", "nom_locate", @@ -1119,8 +1119,8 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "0.26.4" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.5#1f6dc31e2f8ee02cdda255a856d15f253daf17ec" +version = "0.26.6" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.6#955aff48853c6032c75970dc710929f08b4f711a" dependencies = [ "serde_json", ] @@ -1622,8 +1622,8 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "0.26.4" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.5#1f6dc31e2f8ee02cdda255a856d15f253daf17ec" +version = "0.26.6" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.6#955aff48853c6032c75970dc710929f08b4f711a" dependencies = [ "serde_json", ] @@ -2151,8 +2151,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.26.4" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.5#1f6dc31e2f8ee02cdda255a856d15f253daf17ec" +version = "0.26.6" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.6#955aff48853c6032c75970dc710929f08b4f711a" dependencies = [ "bimap", "bincode", diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index 2d9f229f0..f9f5b0edb 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] enum-iterator = "0.7.0" meilisearch-error = { path = "../meilisearch-error" } -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.26.5" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.26.6" } rand = "0.8.4" serde = { version = "1.0.136", features = ["derive"] } serde_json = { version = "1.0.79", features = ["preserve_order"] } diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index 65de9c33d..2da8a8d07 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -30,7 +30,7 @@ lazy_static = "1.4.0" log = "0.4.14" meilisearch-auth = { path = "../meilisearch-auth" } meilisearch-error = { path = "../meilisearch-error" } -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.26.5" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.26.6" } mime = "0.3.16" num_cpus = "1.13.1" obkv = "0.2.0" From 7fd66b80cad396072277c66e3cd46019271220be Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 7 Jun 2022 16:25:47 +0200 Subject: [PATCH 02/23] bump meilisearch version --- Cargo.lock | 8 ++++---- meilisearch-auth/Cargo.toml | 2 +- meilisearch-error/Cargo.toml | 2 +- meilisearch-http/Cargo.toml | 2 +- meilisearch-lib/Cargo.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2b36dbee..81072ecdb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1950,7 +1950,7 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "meilisearch-auth" -version = "0.27.1" +version = "0.27.2" dependencies = [ "enum-iterator", "meilisearch-error", @@ -1965,7 +1965,7 @@ dependencies = [ [[package]] name = "meilisearch-error" -version = "0.27.1" +version = "0.27.2" dependencies = [ "actix-web", "proptest", @@ -1976,7 +1976,7 @@ dependencies = [ [[package]] name = "meilisearch-http" -version = "0.27.1" +version = "0.27.2" dependencies = [ "actix-cors", "actix-rt", @@ -2048,7 +2048,7 @@ dependencies = [ [[package]] name = "meilisearch-lib" -version = "0.27.1" +version = "0.27.2" dependencies = [ "actix-rt", "actix-web", diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index f9f5b0edb..27586db98 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-auth" -version = "0.27.1" +version = "0.27.2" edition = "2021" [dependencies] diff --git a/meilisearch-error/Cargo.toml b/meilisearch-error/Cargo.toml index e47e46b47..0d1bd1d3a 100644 --- a/meilisearch-error/Cargo.toml +++ b/meilisearch-error/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-error" -version = "0.27.1" +version = "0.27.2" authors = ["marin "] edition = "2021" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 0a248f702..4711df2be 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "Meilisearch HTTP server" edition = "2021" license = "MIT" name = "meilisearch-http" -version = "0.27.1" +version = "0.27.2" [[bin]] name = "meilisearch" diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index 2da8a8d07..3d3062e65 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-lib" -version = "0.27.1" +version = "0.27.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 22e1ac969a5fbf5feb370ee59fa1a11b6fb76203 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 15 Jun 2022 15:27:06 +0200 Subject: [PATCH 03/23] Add specific routes for the pagination and faceting settings --- meilisearch-http/src/routes/indexes/settings.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/indexes/settings.rs b/meilisearch-http/src/routes/indexes/settings.rs index 962fe7d82..5b659a323 100644 --- a/meilisearch-http/src/routes/indexes/settings.rs +++ b/meilisearch-http/src/routes/indexes/settings.rs @@ -349,7 +349,9 @@ generate_configure!( stop_words, synonyms, ranking_rules, - typo_tolerance + typo_tolerance, + pagination, + faceting ); pub async fn update_all( @@ -409,6 +411,18 @@ pub async fn update_all( .map(|s| s.two_typos.set())) .flatten(), }, + "faceting": { + "max_values_per_facet": settings.faceting + .as_ref() + .set() + .and_then(|s| s.max_values_per_facet.as_ref().set()), + }, + "pagination": { + "limited_to": settings.pagination + .as_ref() + .set() + .and_then(|s| s.limited_to.as_ref().set()), + }, }), Some(&req), ); From 9d692ba1c687dee0f8da35d484b7b665d55d53a8 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 15 Jun 2022 15:53:43 +0200 Subject: [PATCH 04/23] Add more tests for the pagination and faceting subsettings routes --- .../tests/settings/get_settings.rs | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 0862b15c5..f0c28385d 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -27,7 +27,13 @@ static DEFAULT_SETTINGS_VALUES: Lazy> = Lazy::new(| map.insert( "faceting", json!({ - "maxValuesByFacet": json!(100), + "maxValuesPerFacet": json!(100), + }), + ); + map.insert( + "pagination", + json!({ + "limitedTo": json!(1000), }), ); map @@ -206,7 +212,7 @@ async fn error_update_setting_unexisting_index_invalid_uid() { } macro_rules! test_setting_routes { - ($($setting:ident), *) => { + ($($setting:ident $write_method:ident), *) => { $( mod $setting { use crate::common::Server; @@ -232,7 +238,7 @@ macro_rules! test_setting_routes { .chars() .map(|c| if c == '_' { '-' } else { c }) .collect::()); - let (response, code) = server.service.put(url, serde_json::Value::Null).await; + let (response, code) = server.service.$write_method(url, serde_json::Value::Null).await; assert_eq!(code, 202, "{}", response); server.index("").wait_task(0).await; let (response, code) = server.index("test").get().await; @@ -276,13 +282,15 @@ macro_rules! test_setting_routes { } test_setting_routes!( - filterable_attributes, - displayed_attributes, - searchable_attributes, - distinct_attribute, - stop_words, - ranking_rules, - synonyms + filterable_attributes put, + displayed_attributes put, + searchable_attributes put, + distinct_attribute put, + stop_words put, + ranking_rules put, + synonyms put, + pagination patch, + faceting patch ); #[actix_rt::test] From 10f315015068daeefea3fde91e01730a1b3ab1b5 Mon Sep 17 00:00:00 2001 From: Janith Petangoda <22471198+janithpet@users.noreply.github.com> Date: Wed, 15 Jun 2022 09:45:18 +0100 Subject: [PATCH 05/23] Improve docker CI: push `vX.Y` tag (without patch) to DockerHub (#2507) * Create a docker tag without patch version if git tag has 0 patch version. * Create Docker tag without patch number if git tag follows v.. Add minor changes on CI --- .github/workflows/publish-docker-images.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index b9ea50cb3..8d24c1123 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -27,6 +27,20 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Check tag format + id: check-tag-format + run: | + # Escape submitted tag name + escaped_tag=$(printf "%q" ${{ github.ref_name }}) + + # Check if tag has format v.. and set output.match + # to create a vX.Y (without patch version) Docker tag + if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=match::true + else + echo ::set-output name=match::false + fi + - name: Docker meta id: meta uses: docker/metadata-action@v3 @@ -37,6 +51,7 @@ jobs: flavor: latest=false tags: | type=ref,event=tag + type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.match }} type=raw,value=latest,enable=${{ github.event_name == 'release' }} - name: Build and push From 2dde6fadb4f20610c07e47ecae0f9c397b7e83c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 16 Jun 2022 19:27:27 +0200 Subject: [PATCH 06/23] Check the version in Cargo.toml before publishing --- .github/scripts/check-release.sh | 28 ++++++++++++ .github/workflows/publish-binaries.yml | 10 +++++ .github/workflows/publish-deb-brew-pkg.yml | 10 +++++ .github/workflows/publish-docker-images.yml | 47 ++++++++++++--------- 4 files changed, 75 insertions(+), 20 deletions(-) create mode 100644 .github/scripts/check-release.sh diff --git a/.github/scripts/check-release.sh b/.github/scripts/check-release.sh new file mode 100644 index 000000000..230c3234f --- /dev/null +++ b/.github/scripts/check-release.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# Checking if current tag matches the package version +current_tag=$(echo $GITHUB_REF | tr -d 'refs/tags/v') +file1='meilisearch-auth/Cargo.toml' +file2='meilisearch-http/Cargo.toml' +file3='meilisearch-lib/Cargo.toml' +file4='meilisearch-types/Cargo.toml' +file5='Cargo.lock' + +file_tag1=$(grep '^version = ' $file1 | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') +file_tag2=$(grep '^version = ' $file2 | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') +file_tag3=$(grep '^version = ' $file3 | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') +file_tag4=$(grep '^version = ' $file4 | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') +file_tag5=$(grep -A 1 'name = "meilisearch-auth"' $file5 | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') + +if [ "$current_tag" != "$file_tag1" ] || [ "$current_tag" != "$file_tag2" ] || [ "$current_tag" != "$file_tag3" ] || [ "$current_tag" != "$file_tag4" ] || [ "$current_tag" != "$file_tag5" ]; then + echo "Error: the current tag does not match the version in package file(s)." + echo "$file1: found $file_tag1 - expected $current_tag" + echo "$file2: found $file_tag2 - expected $current_tag" + echo "$file3: found $file_tag3 - expected $current_tag" + echo "$file4: found $file_tag4 - expected $current_tag" + echo "$file5: found $file_tag5 - expected $current_tag" + exit 1 +fi + +echo 'OK' +exit 0 diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 304798d75..eee7449a8 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -5,9 +5,18 @@ on: name: Publish binaries to release jobs: + check-version: + name: Check the version validity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Check release validity + run: sh .github/scripts/check-release.sh + publish: name: Publish binary for ${{ matrix.os }} runs-on: ${{ matrix.os }} + needs: check-version strategy: fail-fast: false matrix: @@ -41,6 +50,7 @@ jobs: publish-aarch64: name: Publish binary for aarch64 runs-on: ${{ matrix.os }} + needs: check-version continue-on-error: false strategy: fail-fast: false diff --git a/.github/workflows/publish-deb-brew-pkg.yml b/.github/workflows/publish-deb-brew-pkg.yml index 6a5a21287..7618496e9 100644 --- a/.github/workflows/publish-deb-brew-pkg.yml +++ b/.github/workflows/publish-deb-brew-pkg.yml @@ -5,9 +5,18 @@ on: types: [released] jobs: + check-version: + name: Check the version validity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Check release validity + run: sh .github/scripts/check-release.sh + debian: name: Publish debian packagge runs-on: ubuntu-18.04 + needs: check-version steps: - uses: hecrj/setup-rust-action@master with: @@ -30,6 +39,7 @@ jobs: homebrew: name: Bump Homebrew formula runs-on: ubuntu-18.04 + needs: check-version steps: - name: Create PR to Homebrew uses: mislav/bump-homebrew-formula-action@v1 diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index 8d24c1123..223ef41b3 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -5,8 +5,6 @@ on: push: tags: - '*' - release: - types: [released] name: Publish tagged images to Docker Hub @@ -14,45 +12,54 @@ jobs: docker: runs-on: docker steps: + - uses: actions/checkout@v2 + + # Check if the tag has the v.. format. If yes, it means we are publishing an official release. + # In this situation, we need to set `output.stable` to create/update the following tags (additionally to the `vX.Y.Z` Docker tag): + # - a `vX.Y` (without patch version) Docker tag + # - a `latest` Docker tag + - name: Check tag format + if: github.event_name != 'schedule' + id: check-tag-format + run: | + escaped_tag=$(printf "%q" ${{ github.ref_name }}) + + if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=stable::true + else + echo ::set-output name=stable::false + fi + + # Check only the validity of the tag for official releases (not for pre-releases or other tags) + - name: Check release validity + if: github.event_name != 'schedule' && steps.check-tag-format.outputs.stable + run: sh .github/scripts/check-release.sh + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub + - name: Login to Docker Hub if: github.event_name != 'schedule' uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Check tag format - id: check-tag-format - run: | - # Escape submitted tag name - escaped_tag=$(printf "%q" ${{ github.ref_name }}) - - # Check if tag has format v.. and set output.match - # to create a vX.Y (without patch version) Docker tag - if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true - else - echo ::set-output name=match::false - fi - - name: Docker meta id: meta uses: docker/metadata-action@v3 with: images: getmeili/meilisearch - # The lastest tag is only pushed for the official Meilisearch release + # The lastest and `vX.Y` tags are only pushed for the official Meilisearch releases # See https://github.com/docker/metadata-action#latest-tag flavor: latest=false tags: | type=ref,event=tag - type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.match }} - type=raw,value=latest,enable=${{ github.event_name == 'release' }} + type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable }} + type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable }} - name: Build and push id: docker_build From 5318e53248eccd0efdc36bfcd2d0c3a4b061a541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 21 Jun 2022 10:08:07 +0200 Subject: [PATCH 07/23] Move is-latest-release.sh script into the scripts folder --- .github/{ => scripts}/is-latest-release.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) rename .github/{ => scripts}/is-latest-release.sh (90%) diff --git a/.github/is-latest-release.sh b/.github/scripts/is-latest-release.sh similarity index 90% rename from .github/is-latest-release.sh rename to .github/scripts/is-latest-release.sh index 0c1db61c2..af0ff45b3 100644 --- a/.github/is-latest-release.sh +++ b/.github/scripts/is-latest-release.sh @@ -1,14 +1,14 @@ #!/bin/sh -# Checks if the current tag should be the latest (in terms of semver and not of release date). -# Ex: previous tag -> v0.10.1 -# new tag -> v0.8.12 -# The new tag should not be the latest -# So it returns "false", the CI should not run for the release v0.8.2 - -# Used in GHA in publish-docker-latest.yml +# Was used in our CIs to publish the latest docker image. Not used anymore, will be used again when v1 and v2 will be out and we will want to maintain multiple stable versions. # Returns "true" or "false" (as a string) to be used in the `if` in GHA +# Checks if the current tag should be the latest (in terms of semver and not of release date). +# Ex: previous tag -> v2.1.1 +# new tag -> v1.20.3 +# The new tag (v1.20.3) should NOT be the latest +# So it returns "false", the `latest tag` should not be updated for the release v1.20.3 and still need to correspond to v2.1.1 + # GLOBAL GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number] From c484d2864617234807541433e75567f9ac9d0949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar=20-=20curqui?= Date: Tue, 21 Jun 2022 10:14:17 +0200 Subject: [PATCH 08/23] Update .github/scripts/check-release.sh Co-authored-by: Tamo --- .github/scripts/check-release.sh | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/scripts/check-release.sh b/.github/scripts/check-release.sh index 230c3234f..ea138c4d7 100644 --- a/.github/scripts/check-release.sh +++ b/.github/scripts/check-release.sh @@ -8,21 +8,18 @@ file3='meilisearch-lib/Cargo.toml' file4='meilisearch-types/Cargo.toml' file5='Cargo.lock' -file_tag1=$(grep '^version = ' $file1 | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') -file_tag2=$(grep '^version = ' $file2 | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') -file_tag3=$(grep '^version = ' $file3 | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') -file_tag4=$(grep '^version = ' $file4 | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') -file_tag5=$(grep -A 1 'name = "meilisearch-auth"' $file5 | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') -if [ "$current_tag" != "$file_tag1" ] || [ "$current_tag" != "$file_tag2" ] || [ "$current_tag" != "$file_tag3" ] || [ "$current_tag" != "$file_tag4" ] || [ "$current_tag" != "$file_tag5" ]; then - echo "Error: the current tag does not match the version in package file(s)." - echo "$file1: found $file_tag1 - expected $current_tag" - echo "$file2: found $file_tag2 - expected $current_tag" - echo "$file3: found $file_tag3 - expected $current_tag" - echo "$file4: found $file_tag4 - expected $current_tag" - echo "$file5: found $file_tag5 - expected $current_tag" - exit 1 -fi +file5=$(grep -A 1 'name = "meilisearch-auth"' $file5 | grep version) + +for file in $file1 $file2 $file3 $file4 $file5; +do + file_tag=$(grep '^version = ' $file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') + if [ "$current_tag" != "$file_tag" ]; then + echo "Error: the current tag does not match the version in package file(s)." + echo "$file: found $file_tag - expected $current_tag" + exit 1 + fi +done echo 'OK' exit 0 From de16de20f46390efe9619bf3788dd802a7a091f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar=20-=20curqui?= Date: Tue, 21 Jun 2022 10:14:24 +0200 Subject: [PATCH 09/23] Update .github/scripts/check-release.sh Co-authored-by: Tamo --- .github/scripts/check-release.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/scripts/check-release.sh b/.github/scripts/check-release.sh index ea138c4d7..189019ce1 100644 --- a/.github/scripts/check-release.sh +++ b/.github/scripts/check-release.sh @@ -2,11 +2,8 @@ # Checking if current tag matches the package version current_tag=$(echo $GITHUB_REF | tr -d 'refs/tags/v') -file1='meilisearch-auth/Cargo.toml' -file2='meilisearch-http/Cargo.toml' -file3='meilisearch-lib/Cargo.toml' -file4='meilisearch-types/Cargo.toml' -file5='Cargo.lock' +files='*/Cargo.toml' +lock_file='Cargo.lock' file5=$(grep -A 1 'name = "meilisearch-auth"' $file5 | grep version) From c6ed756dbc3573c385bb6ed14e9da49f20e37c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 21 Jun 2022 10:46:32 +0200 Subject: [PATCH 10/23] Update script after review --- .github/scripts/check-release.sh | 34 ++++++++++++++++------------ .github/scripts/is-latest-release.sh | 2 +- Cargo.lock | 2 +- permissive-json-pointer/Cargo.toml | 2 +- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/scripts/check-release.sh b/.github/scripts/check-release.sh index 189019ce1..bad957cff 100644 --- a/.github/scripts/check-release.sh +++ b/.github/scripts/check-release.sh @@ -1,22 +1,26 @@ #!/bin/sh -# Checking if current tag matches the package version -current_tag=$(echo $GITHUB_REF | tr -d 'refs/tags/v') -files='*/Cargo.toml' -lock_file='Cargo.lock' - - -file5=$(grep -A 1 'name = "meilisearch-auth"' $file5 | grep version) - -for file in $file1 $file2 $file3 $file4 $file5; -do - file_tag=$(grep '^version = ' $file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') - if [ "$current_tag" != "$file_tag" ]; then - echo "Error: the current tag does not match the version in package file(s)." - echo "$file: found $file_tag - expected $current_tag" +# check_tag $current_tag $file_tag $file_name +function check_tag { + if [ "$1" != "$2" ]; then + echo "Error: the current tag does not match the version in $3:" + echo "Found $1 - expected $2" exit 1 - fi + fi +} + +current_tag=$(echo $GITHUB_REF | tr -d 'refs/tags/v') + +files='*/Cargo.toml' +for file in $files; +do + file_tag="$(grep '^version = ' $file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')" + check_tag $current_tag $file_tag $file done +lock_file='Cargo.lock' +lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') +check_tag $current_tag $lock_tag $lock_file + echo 'OK' exit 0 diff --git a/.github/scripts/is-latest-release.sh b/.github/scripts/is-latest-release.sh index af0ff45b3..81534a2f7 100644 --- a/.github/scripts/is-latest-release.sh +++ b/.github/scripts/is-latest-release.sh @@ -7,7 +7,7 @@ # Ex: previous tag -> v2.1.1 # new tag -> v1.20.3 # The new tag (v1.20.3) should NOT be the latest -# So it returns "false", the `latest tag` should not be updated for the release v1.20.3 and still need to correspond to v2.1.1 +# So it returns "false", the `latest` tag should not be updated for the release v1.20.3 and still need to correspond to v2.1.1 # GLOBAL GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number] diff --git a/Cargo.lock b/Cargo.lock index 1bd47e355..ff2325361 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2515,7 +2515,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "permissive-json-pointer" -version = "0.2.0" +version = "0.28.0" dependencies = [ "big_s", "serde_json", diff --git a/permissive-json-pointer/Cargo.toml b/permissive-json-pointer/Cargo.toml index b50f30f19..9e01b81ab 100644 --- a/permissive-json-pointer/Cargo.toml +++ b/permissive-json-pointer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "permissive-json-pointer" -version = "0.2.0" +version = "0.28.0" edition = "2021" description = "A permissive json pointer" readme = "README.md" From 7490383d4f71d282a075cf3e9421d596c59b51c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 21 Jun 2022 19:17:33 +0200 Subject: [PATCH 11/23] Update the not-released version in Cargo.toml files --- Cargo.lock | 6 +++--- meilisearch-lib/Cargo.toml | 2 +- meilisearch-types/Cargo.toml | 2 +- permissive-json-pointer/Cargo.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff2325361..f736ebecb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2092,7 +2092,7 @@ dependencies = [ [[package]] name = "meilisearch-lib" -version = "0.28.0" +version = "0.0.0" dependencies = [ "actix-rt", "actix-web", @@ -2154,7 +2154,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "0.28.0" +version = "0.0.0" dependencies = [ "actix-web", "proptest", @@ -2515,7 +2515,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "permissive-json-pointer" -version = "0.28.0" +version = "0.0.0" dependencies = [ "big_s", "serde_json", diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index 094c79901..d9603e1d5 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-lib" -version = "0.28.0" +version = "0.0.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 6949722e7..1614f5b34 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-types" -version = "0.28.0" +version = "0.0.0" authors = ["marin "] edition = "2021" diff --git a/permissive-json-pointer/Cargo.toml b/permissive-json-pointer/Cargo.toml index 9e01b81ab..9e01f8807 100644 --- a/permissive-json-pointer/Cargo.toml +++ b/permissive-json-pointer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "permissive-json-pointer" -version = "0.28.0" +version = "0.0.0" edition = "2021" description = "A permissive json pointer" readme = "README.md" From 32c8846514dba7ea03b169c3002bf8365dd714ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 22 Jun 2022 12:20:12 +0200 Subject: [PATCH 12/23] Rollback 0.0.0 versionning --- .github/scripts/check-release.sh | 20 +++++++++++--------- Cargo.lock | 6 +++--- meilisearch-lib/Cargo.toml | 2 +- meilisearch-types/Cargo.toml | 2 +- permissive-json-pointer/Cargo.toml | 2 +- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/scripts/check-release.sh b/.github/scripts/check-release.sh index bad957cff..3c8dd64a7 100644 --- a/.github/scripts/check-release.sh +++ b/.github/scripts/check-release.sh @@ -3,24 +3,26 @@ # check_tag $current_tag $file_tag $file_name function check_tag { if [ "$1" != "$2" ]; then - echo "Error: the current tag does not match the version in $3:" - echo "Found $1 - expected $2" - exit 1 + echo "Error: the current tag does not match the version in $3: found $1 - expected $2" + ret=1 fi } +ret=0 current_tag=$(echo $GITHUB_REF | tr -d 'refs/tags/v') -files='*/Cargo.toml' -for file in $files; +toml_files='*/Cargo.toml' +for toml_file in $toml_files; do - file_tag="$(grep '^version = ' $file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')" - check_tag $current_tag $file_tag $file + file_tag="$(grep '^version = ' $toml_file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')" + check_tag $current_tag $file_tag $toml_file done lock_file='Cargo.lock' lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') check_tag $current_tag $lock_tag $lock_file -echo 'OK' -exit 0 +if [ "$ret" -eq 0 ] ; then + echo 'OK' +fi +exit $ret diff --git a/Cargo.lock b/Cargo.lock index f736ebecb..ff2325361 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2092,7 +2092,7 @@ dependencies = [ [[package]] name = "meilisearch-lib" -version = "0.0.0" +version = "0.28.0" dependencies = [ "actix-rt", "actix-web", @@ -2154,7 +2154,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "0.0.0" +version = "0.28.0" dependencies = [ "actix-web", "proptest", @@ -2515,7 +2515,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "permissive-json-pointer" -version = "0.0.0" +version = "0.28.0" dependencies = [ "big_s", "serde_json", diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index d9603e1d5..094c79901 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-lib" -version = "0.0.0" +version = "0.28.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 1614f5b34..6949722e7 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "meilisearch-types" -version = "0.0.0" +version = "0.28.0" authors = ["marin "] edition = "2021" diff --git a/permissive-json-pointer/Cargo.toml b/permissive-json-pointer/Cargo.toml index 9e01f8807..9e01b81ab 100644 --- a/permissive-json-pointer/Cargo.toml +++ b/permissive-json-pointer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "permissive-json-pointer" -version = "0.0.0" +version = "0.28.0" edition = "2021" description = "A permissive json pointer" readme = "README.md" From c47369b502137ae5c754e666580c10d67819d13c Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 22 Jun 2022 12:33:15 +0200 Subject: [PATCH 13/23] fix all the array on the search get route and improve the tests --- Cargo.lock | 22 ++++++------- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/routes/indexes/search.rs | 21 +++++++------ meilisearch-http/tests/common/index.rs | 2 +- meilisearch-http/tests/search/errors.rs | 24 +++++--------- meilisearch-http/tests/search/formatted.rs | 31 ++++++++++--------- 6 files changed, 50 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1bd47e355..1a861c5fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2069,7 +2069,6 @@ dependencies = [ "serde", "serde-cs", "serde_json", - "serde_url_params", "sha-1", "sha2", "siphasher", @@ -2087,6 +2086,7 @@ dependencies = [ "uuid", "vergen", "walkdir", + "yaup", "zip", ] @@ -3113,16 +3113,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_url_params" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c43307d0640738af32fe8d01e47119bc0fc8a686be470a44a586caff76dfb34" -dependencies = [ - "serde", - "url", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3994,6 +3984,16 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d12cb7a57bbf2ab670ed9545bae3648048547f9039279a89ce000208e585c1" +[[package]] +name = "yaup" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8e1d3d18db742c8b9ad2f5f3c5bf5b63aa67b9933617c8f8350d39a3c173c6" +dependencies = [ + "serde", + "url", +] + [[package]] name = "zerocopy" version = "0.3.0" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 763337888..0012ea9b0 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -83,8 +83,8 @@ actix-rt = "2.7.0" assert-json-diff = "2.0.1" manifest-dir-macros = "0.1.14" maplit = "1.0.2" -serde_url_params = "0.2.1" urlencoding = "2.1.0" +yaup = "0.1.0" [features] default = ["analytics", "mini-dashboard"] diff --git a/meilisearch-http/src/routes/indexes/search.rs b/meilisearch-http/src/routes/indexes/search.rs index 3f8fecd5c..9e3a64881 100644 --- a/meilisearch-http/src/routes/indexes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -14,7 +14,6 @@ use serde_json::Value; use crate::analytics::{Analytics, SearchAggregator}; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::sequential_extractor::SeqHandler; -use crate::routes::{fold_star_or, StarOr}; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( @@ -30,16 +29,16 @@ pub struct SearchQueryGet { q: Option, offset: Option, limit: Option, - attributes_to_retrieve: Option>>, - attributes_to_crop: Option>>, + attributes_to_retrieve: Option>, + attributes_to_crop: Option>, #[serde(default = "DEFAULT_CROP_LENGTH")] crop_length: usize, - attributes_to_highlight: Option>>, + attributes_to_highlight: Option>, filter: Option, sort: Option, #[serde(default = "Default::default")] show_matches_position: bool, - facets: Option>>, + facets: Option>, #[serde(default = "DEFAULT_HIGHLIGHT_PRE_TAG")] highlight_pre_tag: String, #[serde(default = "DEFAULT_HIGHLIGHT_POST_TAG")] @@ -62,14 +61,18 @@ impl From for SearchQuery { q: other.q, offset: other.offset, limit: other.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT), - attributes_to_retrieve: other.attributes_to_retrieve.and_then(fold_star_or), - attributes_to_crop: other.attributes_to_crop.and_then(fold_star_or), + attributes_to_retrieve: other + .attributes_to_retrieve + .map(|o| o.into_iter().collect()), + attributes_to_crop: other.attributes_to_crop.map(|o| o.into_iter().collect()), crop_length: other.crop_length, - attributes_to_highlight: other.attributes_to_highlight.and_then(fold_star_or), + attributes_to_highlight: other + .attributes_to_highlight + .map(|o| o.into_iter().collect()), filter, sort: other.sort.map(|attr| fix_sort_query_parameters(&attr)), show_matches_position: other.show_matches_position, - facets: other.facets.and_then(fold_star_or), + facets: other.facets.map(|o| o.into_iter().collect()), highlight_pre_tag: other.highlight_pre_tag, highlight_post_tag: other.highlight_post_tag, crop_marker: other.crop_marker, diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 010535e21..c92c58560 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -221,7 +221,7 @@ impl Index<'_> { } pub async fn search_get(&self, query: Value) -> (Value, StatusCode) { - let params = serde_url_params::to_string(&query).unwrap(); + let params = yaup::to_string(&query).unwrap(); let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params); self.service.get(url).await } diff --git a/meilisearch-http/tests/search/errors.rs b/meilisearch-http/tests/search/errors.rs index 500825364..c2523597d 100644 --- a/meilisearch-http/tests/search/errors.rs +++ b/meilisearch-http/tests/search/errors.rs @@ -45,26 +45,18 @@ async fn search_invalid_highlight_and_crop_tags() { for field in fields { // object - index - .search( - json!({field.to_string(): {"marker": ""}}), - |response, code| { - assert_eq!(code, 400, "field {} passing object: {}", &field, response); - assert_eq!(response["code"], "bad_request"); - }, - ) + let (response, code) = index + .search_post(json!({field.to_string(): {"marker": ""}})) .await; + assert_eq!(code, 400, "field {} passing object: {}", &field, response); + assert_eq!(response["code"], "bad_request"); // array - index - .search( - json!({field.to_string(): ["marker", ""]}), - |response, code| { - assert_eq!(code, 400, "field {} passing array: {}", &field, response); - assert_eq!(response["code"], "bad_request"); - }, - ) + let (response, code) = index + .search_post(json!({field.to_string(): ["marker", ""]})) .await; + assert_eq!(code, 400, "field {} passing array: {}", &field, response); + assert_eq!(response["code"], "bad_request"); } } diff --git a/meilisearch-http/tests/search/formatted.rs b/meilisearch-http/tests/search/formatted.rs index 9876bac3a..556b0bf35 100644 --- a/meilisearch-http/tests/search/formatted.rs +++ b/meilisearch-http/tests/search/formatted.rs @@ -15,20 +15,23 @@ async fn formatted_contain_wildcard() { index.add_documents(documents, None).await; index.wait_task(1).await; - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["father", "mother"], "attributesToHighlight": ["father", "mother", "*"], "attributesToCrop": ["doggos"], "showMatchesPosition": true })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "id": "852", - "cattos": "pesti", - }, - "_matchesPosition": {"cattos": [{"start": 0, "length": 5}]}, - }) - ); + index.search(json!({ "q": "pesti", "attributesToRetrieve": ["father", "mother"], "attributesToHighlight": ["father", "mother", "*"], "attributesToCrop": ["doggos"], "showMatchesPosition": true }), + |response, code| + { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "id": "852", + "cattos": "pesti", + }, + "_matchesPosition": {"cattos": [{"start": 0, "length": 5}]}, + }) + ); + } + ) + .await; let (response, code) = index .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["*"] })) From 7feb15df2806a7f50c245420a33770fed6f6fe4b Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Thu, 23 Jun 2022 10:47:48 +0200 Subject: [PATCH 14/23] Bump milli to 0.31.1 --- Cargo.lock | 16 ++++++++-------- meilisearch-auth/Cargo.toml | 2 +- meilisearch-lib/Cargo.toml | 4 +--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff2325361..d8bc4fe64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1123,8 +1123,8 @@ dependencies = [ [[package]] name = "filter-parser" -version = "0.29.3" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.29.3#f1d848bb9add86b9414d110a083dfa0462d5d636" +version = "0.31.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8" dependencies = [ "nom", "nom_locate", @@ -1148,8 +1148,8 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "0.29.3" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.29.3#f1d848bb9add86b9414d110a083dfa0462d5d636" +version = "0.31.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8" dependencies = [ "serde_json", ] @@ -1661,8 +1661,8 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "0.29.3" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.29.3#f1d848bb9add86b9414d110a083dfa0462d5d636" +version = "0.31.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8" dependencies = [ "serde_json", ] @@ -2189,8 +2189,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.29.3" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.29.3#f1d848bb9add86b9414d110a083dfa0462d5d636" +version = "0.31.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8" dependencies = [ "bimap", "bincode", diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index ed3a589e2..b4921655e 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -8,7 +8,7 @@ base64 = "0.13.0" enum-iterator = "0.7.0" hmac = "0.12.1" meilisearch-types = { path = "../meilisearch-types" } -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.29.3" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.31.1" } rand = "0.8.4" serde = { version = "1.0.136", features = ["derive"] } serde_json = { version = "1.0.79", features = ["preserve_order"] } diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index 094c79901..3629f51a8 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -3,8 +3,6 @@ name = "meilisearch-lib" version = "0.28.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] actix-web = { version = "4.0.1", default-features = false } anyhow = { version = "1.0.56", features = ["backtrace"] } @@ -30,7 +28,7 @@ lazy_static = "1.4.0" log = "0.4.14" meilisearch-auth = { path = "../meilisearch-auth" } meilisearch-types = { path = "../meilisearch-types" } -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.29.3" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.31.1" } mime = "0.3.16" num_cpus = "1.13.1" obkv = "0.2.0" From dad86fc3d6f87cc37ddb0546a355bb7a1e60a55a Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 22 Jun 2022 17:24:25 +0200 Subject: [PATCH 15/23] Make the changes necessary to use milli 0.31.1 --- .../src/routes/indexes/settings.rs | 6 ++--- meilisearch-http/tests/dumps/mod.rs | 18 ++++++------- meilisearch-http/tests/search/mod.rs | 4 +-- .../tests/settings/get_settings.rs | 4 +-- meilisearch-lib/src/index/dump.rs | 2 +- meilisearch-lib/src/index/index.rs | 25 +++++++++++-------- meilisearch-lib/src/index/search.rs | 12 ++++----- meilisearch-lib/src/index/updates.rs | 10 ++++---- .../src/index_resolver/meta_store.rs | 9 +++++-- meilisearch-lib/src/lib.rs | 17 ------------- meilisearch-lib/src/snapshot.rs | 5 ++-- 11 files changed, 51 insertions(+), 61 deletions(-) diff --git a/meilisearch-http/src/routes/indexes/settings.rs b/meilisearch-http/src/routes/indexes/settings.rs index 5b659a323..bc8642def 100644 --- a/meilisearch-http/src/routes/indexes/settings.rs +++ b/meilisearch-http/src/routes/indexes/settings.rs @@ -318,7 +318,7 @@ make_setting_route!( "Pagination Updated".to_string(), json!({ "pagination": { - "limited_to": setting.as_ref().and_then(|s| s.limited_to.set()), + "max_total_hits": setting.as_ref().and_then(|s| s.max_total_hits.set()), }, }), Some(req), @@ -418,10 +418,10 @@ pub async fn update_all( .and_then(|s| s.max_values_per_facet.as_ref().set()), }, "pagination": { - "limited_to": settings.pagination + "max_total_hits": settings.pagination .as_ref() .set() - .and_then(|s| s.limited_to.as_ref().set()), + .and_then(|s| s.max_total_hits.as_ref().set()), }, }), Some(&req), diff --git a/meilisearch-http/tests/dumps/mod.rs b/meilisearch-http/tests/dumps/mod.rs index 851f502a9..389f6b480 100644 --- a/meilisearch-http/tests/dumps/mod.rs +++ b/meilisearch-http/tests/dumps/mod.rs @@ -61,7 +61,7 @@ async fn import_dump_v2_movie_raw() { assert_eq!(code, 200); assert_eq!( settings, - json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) + json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -125,7 +125,7 @@ async fn import_dump_v2_movie_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) + json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -189,7 +189,7 @@ async fn import_dump_v2_rubygems_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 }}) + json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 }}) ); let (tasks, code) = index.list_tasks().await; @@ -253,7 +253,7 @@ async fn import_dump_v3_movie_raw() { assert_eq!(code, 200); assert_eq!( settings, - json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) + json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -317,7 +317,7 @@ async fn import_dump_v3_movie_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) + json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -381,7 +381,7 @@ async fn import_dump_v3_rubygems_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) + json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -445,7 +445,7 @@ async fn import_dump_v4_movie_raw() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) + json!({ "displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -509,7 +509,7 @@ async fn import_dump_v4_movie_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) + json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) ); let (tasks, code) = index.list_tasks().await; @@ -573,7 +573,7 @@ async fn import_dump_v4_rubygems_with_settings() { assert_eq!(code, 200); assert_eq!( settings, - json!({ "displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "limitedTo": 1000 } }) + json!({ "displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) ); let (tasks, code) = index.list_tasks().await; diff --git a/meilisearch-http/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs index 02cdc751f..b9c187783 100644 --- a/meilisearch-http/tests/search/mod.rs +++ b/meilisearch-http/tests/search/mod.rs @@ -567,7 +567,7 @@ async fn placeholder_search_is_hard_limited() { .await; index - .update_settings(json!({ "pagination": { "limitedTo": 10_000 } })) + .update_settings(json!({ "pagination": { "maxTotalHits": 10_000 } })) .await; index.wait_task(1).await; @@ -636,7 +636,7 @@ async fn search_is_hard_limited() { .await; index - .update_settings(json!({ "pagination": { "limitedTo": 10_000 } })) + .update_settings(json!({ "pagination": { "maxTotalHits": 10_000 } })) .await; index.wait_task(1).await; diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index f0c28385d..9d10b7820 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -33,7 +33,7 @@ static DEFAULT_SETTINGS_VALUES: Lazy> = Lazy::new(| map.insert( "pagination", json!({ - "limitedTo": json!(1000), + "maxTotalHits": json!(1000), }), ); map @@ -82,7 +82,7 @@ async fn get_settings() { assert_eq!( settings["pagination"], json!({ - "limitedTo": 1000, + "maxTotalHits": 1000, }) ); } diff --git a/meilisearch-lib/src/index/dump.rs b/meilisearch-lib/src/index/dump.rs index e201e738b..c6feb187f 100644 --- a/meilisearch-lib/src/index/dump.rs +++ b/meilisearch-lib/src/index/dump.rs @@ -27,7 +27,7 @@ const DATA_FILE_NAME: &str = "documents.jsonl"; impl Index { pub fn dump(&self, path: impl AsRef) -> Result<()> { // acquire write txn make sure any ongoing write is finished before we start. - let txn = self.env.write_txn()?; + let txn = self.write_txn()?; let path = path.as_ref().join(format!("indexes/{}", self.uuid)); create_dir_all(&path)?; diff --git a/meilisearch-lib/src/index/index.rs b/meilisearch-lib/src/index/index.rs index d4772b73b..518e9ce3e 100644 --- a/meilisearch-lib/src/index/index.rs +++ b/meilisearch-lib/src/index/index.rs @@ -4,9 +4,10 @@ use std::marker::PhantomData; use std::ops::Deref; use std::path::Path; use std::sync::Arc; +use walkdir::WalkDir; use fst::IntoStreamer; -use milli::heed::{EnvOpenOptions, RoTxn}; +use milli::heed::{CompactionOption, EnvOpenOptions, RoTxn}; use milli::update::{IndexerConfig, Setting}; use milli::{obkv_to_json, FieldDistribution, DEFAULT_VALUES_PER_FACET}; use serde::{Deserialize, Serialize}; @@ -14,8 +15,7 @@ use serde_json::{Map, Value}; use time::OffsetDateTime; use uuid::Uuid; -use crate::index::search::DEFAULT_PAGINATION_LIMITED_TO; -use crate::EnvSizer; +use crate::index::search::DEFAULT_PAGINATION_MAX_TOTAL_HITS; use super::error::IndexError; use super::error::Result; @@ -202,9 +202,9 @@ impl Index { }; let pagination = PaginationSettings { - limited_to: Setting::Set( - self.pagination_limited_to(txn)? - .unwrap_or(DEFAULT_PAGINATION_LIMITED_TO), + max_total_hits: Setting::Set( + self.pagination_max_total_hits(txn)? + .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS), ), }; @@ -245,7 +245,7 @@ impl Index { let fields_ids_map = self.fields_ids_map(&txn)?; let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); - let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); + let iter = self.all_documents(&txn)?.skip(offset).take(limit); let mut documents = Vec::new(); @@ -302,7 +302,12 @@ impl Index { } pub fn size(&self) -> u64 { - self.env.size() + WalkDir::new(self.inner.path()) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len()) } pub fn snapshot(&self, path: impl AsRef) -> Result<()> { @@ -310,9 +315,7 @@ impl Index { create_dir_all(&dst)?; dst.push("data.mdb"); let _txn = self.write_txn()?; - self.inner - .env - .copy_to_path(dst, milli::heed::CompactionOption::Enabled)?; + self.inner.copy_to_path(dst, CompactionOption::Enabled)?; Ok(()) } } diff --git a/meilisearch-lib/src/index/search.rs b/meilisearch-lib/src/index/search.rs index 781a5bb66..58bcf7ef4 100644 --- a/meilisearch-lib/src/index/search.rs +++ b/meilisearch-lib/src/index/search.rs @@ -29,7 +29,7 @@ pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "".to_string(); /// The maximimum number of results that the engine /// will be able to return in one search call. -pub const DEFAULT_PAGINATION_LIMITED_TO: usize = 1000; +pub const DEFAULT_PAGINATION_MAX_TOTAL_HITS: usize = 1000; #[derive(Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -91,14 +91,14 @@ impl Index { search.query(query); } - let pagination_limited_to = self - .pagination_limited_to(&rtxn)? - .unwrap_or(DEFAULT_PAGINATION_LIMITED_TO); + let max_total_hits = self + .pagination_max_total_hits(&rtxn)? + .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS); // Make sure that a user can't get more documents than the hard limit, // we align that on the offset too. - let offset = min(query.offset.unwrap_or(0), pagination_limited_to); - let limit = min(query.limit, pagination_limited_to.saturating_sub(offset)); + let offset = min(query.offset.unwrap_or(0), max_total_hits); + let limit = min(query.limit, max_total_hits.saturating_sub(offset)); search.offset(offset); search.limit(limit); diff --git a/meilisearch-lib/src/index/updates.rs b/meilisearch-lib/src/index/updates.rs index 95edbbf9d..07695af05 100644 --- a/meilisearch-lib/src/index/updates.rs +++ b/meilisearch-lib/src/index/updates.rs @@ -86,7 +86,7 @@ pub struct FacetingSettings { pub struct PaginationSettings { #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] #[serde(default, skip_serializing_if = "Setting::is_not_set")] - pub limited_to: Setting, + pub max_total_hits: Setting, } /// Holds all the settings for an index. `T` can either be `Checked` if they represents settings @@ -474,12 +474,12 @@ pub fn apply_settings_to_builder( } match settings.pagination { - Setting::Set(ref value) => match value.limited_to { - Setting::Set(val) => builder.set_pagination_limited_to(val), - Setting::Reset => builder.reset_pagination_limited_to(), + Setting::Set(ref value) => match value.max_total_hits { + Setting::Set(val) => builder.set_pagination_max_total_hits(val), + Setting::Reset => builder.reset_pagination_max_total_hits(), Setting::NotSet => (), }, - Setting::Reset => builder.reset_pagination_limited_to(), + Setting::Reset => builder.reset_pagination_max_total_hits(), Setting::NotSet => (), } } diff --git a/meilisearch-lib/src/index_resolver/meta_store.rs b/meilisearch-lib/src/index_resolver/meta_store.rs index f53f9cae9..f335d9923 100644 --- a/meilisearch-lib/src/index_resolver/meta_store.rs +++ b/meilisearch-lib/src/index_resolver/meta_store.rs @@ -3,6 +3,7 @@ use std::fs::{create_dir_all, File}; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use walkdir::WalkDir; use milli::heed::types::{SerdeBincode, Str}; use milli::heed::{CompactionOption, Database, Env}; @@ -11,7 +12,6 @@ use uuid::Uuid; use super::error::{IndexResolverError, Result}; use crate::tasks::task::TaskId; -use crate::EnvSizer; #[derive(Serialize, Deserialize)] pub struct DumpEntry { @@ -131,7 +131,12 @@ impl HeedMetaStore { } fn get_size(&self) -> Result { - Ok(self.env.size()) + Ok(WalkDir::new(self.env.path()) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len())) } pub fn dump(&self, path: PathBuf) -> Result<()> { diff --git a/meilisearch-lib/src/lib.rs b/meilisearch-lib/src/lib.rs index 3d3d5e860..70fd2ba51 100644 --- a/meilisearch-lib/src/lib.rs +++ b/meilisearch-lib/src/lib.rs @@ -20,23 +20,6 @@ pub use milli::heed; mod compression; pub mod document_formats; -use walkdir::WalkDir; - -pub trait EnvSizer { - fn size(&self) -> u64; -} - -impl EnvSizer for milli::heed::Env { - fn size(&self) -> u64 { - WalkDir::new(self.path()) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| entry.metadata().ok()) - .filter(|metadata| metadata.is_file()) - .fold(0, |acc, m| acc + m.len()) - } -} - /// Check if a db is empty. It does not provide any information on the /// validity of the data in it. /// We consider a database as non empty when it's a non empty directory. diff --git a/meilisearch-lib/src/snapshot.rs b/meilisearch-lib/src/snapshot.rs index 527195729..da4907939 100644 --- a/meilisearch-lib/src/snapshot.rs +++ b/meilisearch-lib/src/snapshot.rs @@ -7,6 +7,7 @@ use anyhow::bail; use fs_extra::dir::{self, CopyOptions}; use log::{info, trace}; use meilisearch_auth::open_auth_store_env; +use milli::heed::CompactionOption; use tokio::sync::RwLock; use tokio::time::sleep; use walkdir::WalkDir; @@ -181,9 +182,7 @@ impl SnapshotJob { let mut options = milli::heed::EnvOpenOptions::new(); options.map_size(self.index_size); let index = milli::Index::new(options, entry.path())?; - index - .env - .copy_to_path(dst, milli::heed::CompactionOption::Enabled)?; + index.copy_to_path(dst, CompactionOption::Enabled)?; } Ok(()) From f83188fd606c9a0d576a2dfeafcaac67b7a0d760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 23 Jun 2022 19:14:39 +0200 Subject: [PATCH 16/23] Fix CI with check-release.sh script --- .github/scripts/check-release.sh | 10 +++++----- .github/workflows/publish-binaries.yml | 2 +- .github/workflows/publish-deb-brew-pkg.yml | 2 +- .github/workflows/publish-docker-images.yml | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/scripts/check-release.sh b/.github/scripts/check-release.sh index 3c8dd64a7..2ce171459 100644 --- a/.github/scripts/check-release.sh +++ b/.github/scripts/check-release.sh @@ -1,15 +1,15 @@ -#!/bin/sh +#!/bin/bash # check_tag $current_tag $file_tag $file_name function check_tag { - if [ "$1" != "$2" ]; then - echo "Error: the current tag does not match the version in $3: found $1 - expected $2" + if [[ "$1" != "$2" ]]; then + echo "Error: the current tag does not match the version in $3: found $2 - expected $1" ret=1 fi } ret=0 -current_tag=$(echo $GITHUB_REF | tr -d 'refs/tags/v') +current_tag=${GITHUB_REF#'refs/tags/v'} toml_files='*/Cargo.toml' for toml_file in $toml_files; @@ -22,7 +22,7 @@ lock_file='Cargo.lock' lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') check_tag $current_tag $lock_tag $lock_file -if [ "$ret" -eq 0 ] ; then +if [[ "$ret" -eq 0 ]] ; then echo 'OK' fi exit $ret diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index eee7449a8..215357f02 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Check release validity - run: sh .github/scripts/check-release.sh + run: bash .github/scripts/check-release.sh publish: name: Publish binary for ${{ matrix.os }} diff --git a/.github/workflows/publish-deb-brew-pkg.yml b/.github/workflows/publish-deb-brew-pkg.yml index 7618496e9..dbdbdda7e 100644 --- a/.github/workflows/publish-deb-brew-pkg.yml +++ b/.github/workflows/publish-deb-brew-pkg.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Check release validity - run: sh .github/scripts/check-release.sh + run: bash .github/scripts/check-release.sh debian: name: Publish debian packagge diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index 223ef41b3..0d2e2b60e 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -32,8 +32,8 @@ jobs: # Check only the validity of the tag for official releases (not for pre-releases or other tags) - name: Check release validity - if: github.event_name != 'schedule' && steps.check-tag-format.outputs.stable - run: sh .github/scripts/check-release.sh + if: github.event_name != 'schedule' && steps.check-tag-format.outputs.stable == 'true' + run: bash .github/scripts/check-release.sh - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -58,8 +58,8 @@ jobs: flavor: latest=false tags: | type=ref,event=tag - type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable }} - type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable }} + type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }} + type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' }} - name: Build and push id: docker_build From 7f4fab876d5e52ce39a7de74919e45da41655123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 27 Jun 2022 13:11:58 +0200 Subject: [PATCH 17/23] Add fix to publish-binaries.yml --- .github/workflows/publish-binaries.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 215357f02..298082816 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -10,7 +10,22 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # Check if the tag has the v.. format. + # If yes, it means we are publishing an official release. + # If no, we are releasing a RC, so no need to check the version. + - name: Check tag format + if: github.event_name != 'schedule' + id: check-tag-format + run: | + escaped_tag=$(printf "%q" ${{ github.ref_name }}) + + if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=stable::true + else + echo ::set-output name=stable::false + fi - name: Check release validity + if: steps.check-tag-format.outputs.stable == 'true' run: bash .github/scripts/check-release.sh publish: From 05ee2eff01ab0d7b429ef0330c3798d0d981ceb0 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Tue, 28 Jun 2022 13:01:18 +0200 Subject: [PATCH 18/23] add more tests on the formatted route --- Cargo.lock | 4 +- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/tests/search/errors.rs | 8 +- meilisearch-http/tests/search/formatted.rs | 599 ++++++++++++--------- 4 files changed, 341 insertions(+), 272 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3af8ab7ad..bc9a4f992 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3986,9 +3986,9 @@ checksum = "b6d12cb7a57bbf2ab670ed9545bae3648048547f9039279a89ce000208e585c1" [[package]] name = "yaup" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8e1d3d18db742c8b9ad2f5f3c5bf5b63aa67b9933617c8f8350d39a3c173c6" +checksum = "3bc9ef6963f7e857050aabf31ebc44184f278bcfec4c3671552c1a916b152b45" dependencies = [ "serde", "url", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 0012ea9b0..b5466d13c 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -84,7 +84,7 @@ assert-json-diff = "2.0.1" manifest-dir-macros = "0.1.14" maplit = "1.0.2" urlencoding = "2.1.0" -yaup = "0.1.0" +yaup = "0.2.0" [features] default = ["analytics", "mini-dashboard"] diff --git a/meilisearch-http/tests/search/errors.rs b/meilisearch-http/tests/search/errors.rs index c2523597d..98da0495a 100644 --- a/meilisearch-http/tests/search/errors.rs +++ b/meilisearch-http/tests/search/errors.rs @@ -107,7 +107,7 @@ async fn filter_invalid_syntax_array() { "link": "https://docs.meilisearch.com/errors#invalid_filter" }); index - .search(json!({"filter": [["title & Glass"]]}), |response, code| { + .search(json!({"filter": ["title & Glass"]}), |response, code| { assert_eq!(response, expected_response); assert_eq!(code, 400); }) @@ -164,7 +164,7 @@ async fn filter_invalid_attribute_array() { "link": "https://docs.meilisearch.com/errors#invalid_filter" }); index - .search(json!({"filter": [["many = Glass"]]}), |response, code| { + .search(json!({"filter": ["many = Glass"]}), |response, code| { assert_eq!(response, expected_response); assert_eq!(code, 400); }) @@ -218,7 +218,7 @@ async fn filter_reserved_geo_attribute_array() { "link": "https://docs.meilisearch.com/errors#invalid_filter" }); index - .search(json!({"filter": [["_geo = Glass"]]}), |response, code| { + .search(json!({"filter": ["_geo = Glass"]}), |response, code| { assert_eq!(response, expected_response); assert_eq!(code, 400); }) @@ -273,7 +273,7 @@ async fn filter_reserved_attribute_array() { }); index .search( - json!({"filter": [["_geoDistance = Glass"]]}), + json!({"filter": ["_geoDistance = Glass"]}), |response, code| { assert_eq!(response, expected_response); assert_eq!(code, 400); diff --git a/meilisearch-http/tests/search/formatted.rs b/meilisearch-http/tests/search/formatted.rs index 556b0bf35..7303a7154 100644 --- a/meilisearch-http/tests/search/formatted.rs +++ b/meilisearch-http/tests/search/formatted.rs @@ -33,70 +33,82 @@ async fn formatted_contain_wildcard() { ) .await; - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["*"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "cattos": "pesti", - }) - ); - - let (response, code) = index - .search_post( - json!({ "q": "pesti", "attributesToRetrieve": ["*"], "attributesToHighlight": ["id"], "showMatchesPosition": true }), - ) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "cattos": "pesti", - "_formatted": { - "id": "852", - "cattos": "pesti", + index + .search( + json!({ "q": "pesti", "attributesToRetrieve": ["*"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "cattos": "pesti", + }) + ); }, - "_matchesPosition": {"cattos": [{"start": 0, "length": 5}]}, - }) - ); - - let (response, code) = index - .search_post( - json!({ "q": "pesti", "attributesToRetrieve": ["*"], "attributesToCrop": ["*"] }), ) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "cattos": "pesti", - "_formatted": { - "id": "852", - "cattos": "pesti", - } - }) - ); - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToCrop": ["*"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "cattos": "pesti", - "_formatted": { - "id": "852", - "cattos": "pesti", + index + .search( + json!({ "q": "pesti", "attributesToRetrieve": ["*"], "attributesToHighlight": ["id"], "showMatchesPosition": true }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "cattos": "pesti", + "_formatted": { + "id": "852", + "cattos": "pesti", + }, + "_matchesPosition": {"cattos": [{"start": 0, "length": 5}]}, + }) + ); } - }) - ); + ) + .await; + + index + .search( + json!({ "q": "pesti", "attributesToRetrieve": ["*"], "attributesToCrop": ["*"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "cattos": "pesti", + "_formatted": { + "id": "852", + "cattos": "pesti", + } + }) + ); + }, + ) + .await; + + index + .search( + json!({ "q": "pesti", "attributesToCrop": ["*"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "cattos": "pesti", + "_formatted": { + "id": "852", + "cattos": "pesti", + } + }) + ); + }, + ) + .await; } #[actix_rt::test] @@ -108,108 +120,122 @@ async fn format_nested() { index.add_documents(documents, None).await; index.wait_task(0).await; - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["doggos"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "doggos": [ - { - "name": "bobby", - "age": 2, - }, - { - "name": "buddy", - "age": 4, - }, - ], - }) - ); - - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "doggos": [ - { - "name": "bobby", - }, - { - "name": "buddy", - }, - ], - }) - ); - - let (response, code) = index - .search_post( - json!({ "q": "bobby", "attributesToRetrieve": ["doggos.name"], "showMatchesPosition": true }), + index + .search( + json!({ "q": "pesti", "attributesToRetrieve": ["doggos"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "doggos": [ + { + "name": "bobby", + "age": 2, + }, + { + "name": "buddy", + "age": 4, + }, + ], + }) + ); + }, ) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "doggos": [ - { - "name": "bobby", - }, - { - "name": "buddy", - }, - ], - "_matchesPosition": {"doggos.name": [{"start": 0, "length": 5}]}, - }) - ); - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.name"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "doggos": [ - { - "name": "bobby", - }, - { - "name": "buddy", - }, - ], + index + .search( + json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "doggos": [ + { + "name": "bobby", + }, + { + "name": "buddy", + }, + ], + }) + ); }, - }) - ); - - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToCrop": ["doggos.name"] })) + ) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "doggos": [ - { - "name": "bobby", - }, - { - "name": "buddy", - }, - ], - }, - }) - ); - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"], "attributesToHighlight": ["doggos.age"] })) + index + .search( + json!({ "q": "bobby", "attributesToRetrieve": ["doggos.name"], "showMatchesPosition": true }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "doggos": [ + { + "name": "bobby", + }, + { + "name": "buddy", + }, + ], + "_matchesPosition": {"doggos.name": [{"start": 0, "length": 5}]}, + }) + ); + } + ) .await; + + index + .search(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.name"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "doggos": [ + { + "name": "bobby", + }, + { + "name": "buddy", + }, + ], + }, + }) + ); + }) + .await; + + index + .search(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToCrop": ["doggos.name"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "doggos": [ + { + "name": "bobby", + }, + { + "name": "buddy", + }, + ], + }, + }) + ); + }) + .await; + + index + .search(json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"], "attributesToHighlight": ["doggos.age"] }), + |response, code| { assert_eq!(code, 200, "{}", response); assert_eq!( response["hits"][0], @@ -236,11 +262,13 @@ async fn format_nested() { }, }) ); - - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.age"], "attributesToCrop": ["doggos.name"] })) + }) .await; - assert_eq!(code, 200, "{}", response); + + index + .search(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.age"], "attributesToCrop": ["doggos.name"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); assert_eq!( response["hits"][0], json!({ @@ -258,6 +286,9 @@ async fn format_nested() { }, }) ); + } + ) + .await; } #[actix_rt::test] @@ -274,9 +305,9 @@ async fn displayedattr_2_smol() { index.add_documents(documents, None).await; index.wait_task(1).await; - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["father", "id"], "attributesToHighlight": ["mother"], "attributesToCrop": ["cattos"] })) - .await; + index + .search(json!({ "attributesToRetrieve": ["father", "id"], "attributesToHighlight": ["mother"], "attributesToCrop": ["cattos"] }), + |response, code| { assert_eq!(code, 200, "{}", response); assert_eq!( response["hits"][0], @@ -284,119 +315,157 @@ async fn displayedattr_2_smol() { "id": 852, }) ); - - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["id"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToHighlight": ["id"] })) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "_formatted": { - "id": "852", - } - }) - ); - let (response, code) = index - .search_post(json!({ "attributesToCrop": ["id"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "_formatted": { - "id": "852", - } - }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToHighlight": ["id"], "attributesToCrop": ["id"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "_formatted": { - "id": "852", - } - }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToHighlight": ["cattos"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToCrop": ["cattos"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["cattos"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!(response["hits"][0], json!({})); - - let (response, code) = index - .search_post( - json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["cattos"], "attributesToCrop": ["cattos"] }), + index + .search( + json!({ "attributesToRetrieve": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + }) + ); + }, ) .await; + + index + .search( + json!({ "attributesToHighlight": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "_formatted": { + "id": "852", + } + }) + ); + }, + ) + .await; + + index + .search(json!({ "attributesToCrop": ["id"] }), |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "_formatted": { + "id": "852", + } + }) + ); + }) + .await; + + index + .search( + json!({ "attributesToHighlight": ["id"], "attributesToCrop": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "_formatted": { + "id": "852", + } + }) + ); + }, + ) + .await; + + index + .search( + json!({ "attributesToHighlight": ["cattos"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + }) + ); + }, + ) + .await; + + index + .search( + json!({ "attributesToCrop": ["cattos"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + }) + ); + }, + ) + .await; + + index + .search( + json!({ "attributesToRetrieve": ["cattos"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"][0], json!({})); + }, + ) + .await; + + index + .search( + json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["cattos"], "attributesToCrop": ["cattos"] }), + |response, code| { assert_eq!(code, 200, "{}", response); assert_eq!(response["hits"][0], json!({})); - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["id"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "id": "852", } - }) - ); + ) + .await; - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["cattos"], "attributesToCrop": ["id"] })) + index + .search( + json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "id": "852", + } + }) + ); + }, + ) + .await; + + index + .search( + json!({ "attributesToRetrieve": ["cattos"], "attributesToCrop": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "id": "852", + } + }) + ); + }, + ) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "id": "852", - } - }) - ); } From a146fd45b91f7cea2e452d2f5e1525ccaa634761 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Tue, 5 Jul 2022 14:13:20 +0200 Subject: [PATCH 19/23] Format API keys in hexa instead of base64 --- Cargo.lock | 18 ++++++++++----- meilisearch-auth/Cargo.toml | 3 +-- meilisearch-auth/src/lib.rs | 4 ++-- meilisearch-auth/src/store.rs | 30 ++++++++++++++++--------- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/tests/auth/api_keys.rs | 1 + meilisearch-lib/Cargo.toml | 2 +- 7 files changed, 38 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc9a4f992..82f83375e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2003,7 +2003,6 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" name = "meilisearch-auth" version = "0.28.0" dependencies = [ - "base64", "enum-iterator", "hmac", "meilisearch-types", @@ -2014,7 +2013,7 @@ dependencies = [ "sha2", "thiserror", "time 0.3.9", - "uuid", + "uuid 1.1.2", ] [[package]] @@ -2083,7 +2082,7 @@ dependencies = [ "tokio", "tokio-stream", "urlencoding", - "uuid", + "uuid 1.1.2", "vergen", "walkdir", "yaup", @@ -2147,7 +2146,7 @@ dependencies = [ "thiserror", "time 0.3.9", "tokio", - "uuid", + "uuid 1.1.2", "walkdir", "whoami", ] @@ -2229,7 +2228,7 @@ dependencies = [ "tempfile", "thiserror", "time 0.3.9", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -3676,6 +3675,15 @@ name = "uuid" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" dependencies = [ "getrandom", "serde", diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index b4921655e..3ba5408e8 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -4,7 +4,6 @@ version = "0.28.0" edition = "2021" [dependencies] -base64 = "0.13.0" enum-iterator = "0.7.0" hmac = "0.12.1" meilisearch-types = { path = "../meilisearch-types" } @@ -15,4 +14,4 @@ serde_json = { version = "1.0.79", features = ["preserve_order"] } sha2 = "0.10.2" thiserror = "1.0.30" time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } -uuid = { version = "0.8.2", features = ["serde", "v4"] } +uuid = { version = "1.1.2", features = ["serde", "v4"] } diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index 81443348a..17f1a3567 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -18,7 +18,7 @@ pub use action::{actions, Action}; use error::{AuthControllerError, Result}; pub use key::Key; use meilisearch_types::star_or::StarOr; -use store::generate_key_as_base64; +use store::generate_key_as_hexa; pub use store::open_auth_store_env; use store::HeedAuthStore; @@ -139,7 +139,7 @@ impl AuthController { pub fn generate_key(&self, uid: Uuid) -> Option { self.master_key .as_ref() - .map(|master_key| generate_key_as_base64(uid.as_bytes(), master_key.as_bytes())) + .map(|master_key| generate_key_as_hexa(uid, master_key.as_bytes())) } /// Check if the provided key is authorized to make a specific action diff --git a/meilisearch-auth/src/store.rs b/meilisearch-auth/src/store.rs index 0355c4579..49bbf356e 100644 --- a/meilisearch-auth/src/store.rs +++ b/meilisearch-auth/src/store.rs @@ -13,8 +13,9 @@ use hmac::{Hmac, Mac}; use meilisearch_types::star_or::StarOr; use milli::heed::types::{ByteSlice, DecodeIgnore, SerdeJson}; use milli::heed::{Database, Env, EnvOpenOptions, RwTxn}; -use sha2::{Digest, Sha256}; +use sha2::Sha256; use time::OffsetDateTime; +use uuid::fmt::Hyphenated; use uuid::Uuid; use super::error::Result; @@ -132,13 +133,16 @@ impl HeedAuthStore { .remap_data_type::() .iter(&rtxn)? .filter_map(|res| match res { - Ok((uid, _)) - if generate_key_as_base64(uid, master_key).as_bytes() == encoded_key => - { + Ok((uid, _)) => { let (uid, _) = try_split_array_at(uid)?; - Some(Uuid::from_bytes(*uid)) + let uid = Uuid::from_bytes(*uid); + if generate_key_as_hexa(uid, master_key).as_bytes() == encoded_key { + Some(uid) + } else { + None + } } - _ => None, + Err(_) => None, }) .next(); @@ -244,13 +248,17 @@ impl<'a> milli::heed::BytesEncode<'a> for KeyIdActionCodec { } } -pub fn generate_key_as_base64(uid: &[u8], master_key: &[u8]) -> String { - let master_key_sha = Sha256::digest(master_key); - let mut mac = Hmac::::new_from_slice(master_key_sha.as_slice()).unwrap(); - mac.update(uid); +pub fn generate_key_as_hexa(uid: Uuid, master_key: &[u8]) -> String { + // format uid as hyphenated allowing user to generate their own keys. + let mut uid_buffer = [0; Hyphenated::LENGTH]; + let uid = uid.hyphenated().encode_lower(&mut uid_buffer); + + // new_from_slice function never fail. + let mut mac = Hmac::::new_from_slice(master_key).unwrap(); + mac.update(uid.as_bytes()); let result = mac.finalize(); - base64::encode_config(result.into_bytes(), base64::URL_SAFE_NO_PAD) + format!("{:x}", result.into_bytes()) } /// Divides one slice into two at an index, returns `None` if mid is out of bounds. diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index b5466d13c..5cea82d6f 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -75,7 +75,7 @@ thiserror = "1.0.30" time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } tokio = { version = "1.17.0", features = ["full"] } tokio-stream = "0.1.8" -uuid = { version = "0.8.2", features = ["serde", "v4"] } +uuid = { version = "1.1.2", features = ["serde", "v4"] } walkdir = "2.3.2" [dev-dependencies] diff --git a/meilisearch-http/tests/auth/api_keys.rs b/meilisearch-http/tests/auth/api_keys.rs index 9dcbd9b55..7fdf2f129 100644 --- a/meilisearch-http/tests/auth/api_keys.rs +++ b/meilisearch-http/tests/auth/api_keys.rs @@ -42,6 +42,7 @@ async fn add_valid_api_key() { "name": "indexing-key", "description": "Indexing API key", "uid": "4bc0887a-0e41-4f3b-935d-0c451dcee9c8", + "key": "d9e776b8412f1db6974c9a5556b961c3559440b6588216f4ea5d9ed49f7c8f3c", "indexes": ["products"], "actions": [ "search", diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index 3629f51a8..3d5505e5f 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -51,7 +51,7 @@ tempfile = "3.3.0" thiserror = "1.0.30" time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } tokio = { version = "1.17.0", features = ["full"] } -uuid = { version = "0.8.2", features = ["serde", "v4"] } +uuid = { version = "1.1.2", features = ["serde", "v4"] } walkdir = "2.3.2" whoami = { version = "1.2.1", optional = true } From d56bf660221f480b46a8e868361d8f9e38dfbbe2 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 4 Jul 2022 12:00:03 +0200 Subject: [PATCH 20/23] Make clippy happy --- meilisearch-http/src/lib.rs | 2 +- meilisearch-http/src/routes/indexes/search.rs | 7 +++---- meilisearch-http/tests/common/index.rs | 13 +++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index bfdb829d4..6485784fc 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -31,7 +31,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result { let mut meilisearch = MeiliSearch::builder(); // enable autobatching? - let _ = AUTOBATCHING_ENABLED.store( + AUTOBATCHING_ENABLED.store( opt.scheduler_options.enable_auto_batching, std::sync::atomic::Ordering::Relaxed, ); diff --git a/meilisearch-http/src/routes/indexes/search.rs b/meilisearch-http/src/routes/indexes/search.rs index 9e3a64881..62bd65e14 100644 --- a/meilisearch-http/src/routes/indexes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -111,10 +111,9 @@ fn fix_sort_query_parameters(sort_query: &str) -> Vec { sort_parameters.push(current_sort.to_string()); merge = true; } else if merge && !sort_parameters.is_empty() { - sort_parameters - .last_mut() - .unwrap() - .push_str(&format!(",{}", current_sort)); + let s = sort_parameters.last_mut().unwrap(); + s.push(','); + s.push_str(current_sort); if current_sort.ends_with("):desc") || current_sort.ends_with("):asc") { merge = false; } diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index c92c58560..90d138ced 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -1,4 +1,5 @@ use std::{ + fmt::Write, panic::{catch_unwind, resume_unwind, UnwindSafe}, time::Duration, }; @@ -118,10 +119,10 @@ impl Index<'_> { pub async fn filtered_tasks(&self, type_: &[&str], status: &[&str]) -> (Value, StatusCode) { let mut url = format!("/tasks?indexUid={}", self.uid); if !type_.is_empty() { - url += &format!("&type={}", type_.join(",")); + let _ = write!(url, "&type={}", type_.join(",")); } if !status.is_empty() { - url += &format!("&status={}", status.join(",")); + let _ = write!(url, "&status={}", status.join(",")); } self.service.get(url).await } @@ -133,7 +134,7 @@ impl Index<'_> { ) -> (Value, StatusCode) { let mut url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); if let Some(fields) = options.and_then(|o| o.fields) { - url.push_str(&format!("?fields={}", fields.join(","))); + let _ = write!(url, "?fields={}", fields.join(",")); } self.service.get(url).await } @@ -141,15 +142,15 @@ impl Index<'_> { pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) { let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref())); if let Some(limit) = options.limit { - url.push_str(&format!("limit={}&", limit)); + let _ = write!(url, "limit={}&", limit); } if let Some(offset) = options.offset { - url.push_str(&format!("offset={}&", offset)); + let _ = write!(url, "offset={}&", offset); } if let Some(attributes_to_retrieve) = options.attributes_to_retrieve { - url.push_str(&format!("fields={}&", attributes_to_retrieve.join(","))); + let _ = write!(url, "fields={}&", attributes_to_retrieve.join(",")); } self.service.get(url).await From 719879d4d28ff9cb93b4c2f0567ab28bef4bbb4f Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Wed, 6 Jul 2022 08:12:17 +0200 Subject: [PATCH 21/23] Update mini-dashboard to v0.2.0 --- meilisearch-http/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index b5466d13c..566c3e2f3 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -105,5 +105,5 @@ mini-dashboard = [ tikv-jemallocator = "0.4.3" [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.10/build.zip" -sha1 = "1adf96592c267425c110bfefc36b7fc6bfb0f93d" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.0/build.zip" +sha1 = "25d1615c608541375a08bd722c3fd3315f926be6" From a9fb5a4d504febf5f6e75924a4db339505f8c398 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Thu, 7 Jul 2022 10:56:02 +0200 Subject: [PATCH 22/23] Introduce the Tasks Seen event when filtering --- .../src/analytics/segment_analytics.rs | 8 +++---- meilisearch-http/src/routes/tasks.rs | 22 +++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/meilisearch-http/src/analytics/segment_analytics.rs b/meilisearch-http/src/analytics/segment_analytics.rs index 1007a242a..b04d814aa 100644 --- a/meilisearch-http/src/analytics/segment_analytics.rs +++ b/meilisearch-http/src/analytics/segment_analytics.rs @@ -574,7 +574,7 @@ impl DocumentsAggregator { let content_type = request .headers() .get(CONTENT_TYPE) - .map(|s| s.to_str().unwrap_or("unknown")) + .and_then(|s| s.to_str().ok()) .unwrap_or("unknown") .to_string(); ret.content_types.insert(content_type); @@ -591,13 +591,13 @@ impl DocumentsAggregator { self.updated |= other.updated; // we can't create a union because there is no `into_union` method - for user_agent in other.user_agents.into_iter() { + for user_agent in other.user_agents { self.user_agents.insert(user_agent); } - for primary_key in other.primary_keys.into_iter() { + for primary_key in other.primary_keys { self.primary_keys.insert(primary_key); } - for content_type in other.content_types.into_iter() { + for content_type in other.content_types { self.content_types.insert(content_type); } self.index_creation |= other.index_creation; diff --git a/meilisearch-http/src/routes/tasks.rs b/meilisearch-http/src/routes/tasks.rs index fed7fa634..016aadfb8 100644 --- a/meilisearch-http/src/routes/tasks.rs +++ b/meilisearch-http/src/routes/tasks.rs @@ -25,7 +25,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct TaskFilterQuery { +pub struct TasksFilterQuery { #[serde(rename = "type")] type_: Option>>, status: Option>>, @@ -61,17 +61,11 @@ fn task_status_matches_events(status: &TaskStatus, events: &[TaskEvent]) -> bool async fn get_tasks( meilisearch: GuardedData, MeiliSearch>, - params: web::Query, + params: web::Query, req: HttpRequest, analytics: web::Data, ) -> Result { - analytics.publish( - "Tasks Seen".to_string(), - json!({ "per_task_uid": false }), - Some(&req), - ); - - let TaskFilterQuery { + let TasksFilterQuery { type_, status, index_uid, @@ -87,6 +81,16 @@ async fn get_tasks( let status: Option> = status.and_then(fold_star_or); let index_uid: Option> = index_uid.and_then(fold_star_or); + analytics.publish( + "Tasks Seen".to_string(), + json!({ + "filtered_by_index_uid": index_uid.as_ref().map_or(false, |v| !v.is_empty()), + "filtered_by_type": type_.as_ref().map_or(false, |v| !v.is_empty()), + "filtered_by_status": status.as_ref().map_or(false, |v| !v.is_empty()), + }), + Some(&req), + ); + // Then we filter on potential indexes and make sure that the search filter // restrictions are also applied. let indexes_filters = match index_uid { From cfacc79ad7811a906b1b02cb81086bf40ae9e813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 11 Jul 2022 15:18:15 +0200 Subject: [PATCH 23/23] Remove duplicate step in CI --- .github/workflows/publish-docker-images.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index f3c6c6268..72234fc01 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -48,20 +48,6 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Check tag format - id: check-tag-format - run: | - # Escape submitted tag name - escaped_tag=$(printf "%q" ${{ github.ref_name }}) - - # Check if tag has format v.. and set output.match - # to create a vX.Y (without patch version) Docker tag - if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true - else - echo ::set-output name=match::false - fi - - name: Docker meta id: meta uses: docker/metadata-action@v4