diff --git a/.github/scripts/check-release.sh b/.github/scripts/check-release.sh new file mode 100644 index 000000000..2ce171459 --- /dev/null +++ b/.github/scripts/check-release.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# check_tag $current_tag $file_tag $file_name +function check_tag { + if [[ "$1" != "$2" ]]; then + echo "Error: the current tag does not match the version in $3: found $2 - expected $1" + ret=1 + fi +} + +ret=0 +current_tag=${GITHUB_REF#'refs/tags/v'} + +toml_files='*/Cargo.toml' +for toml_file in $toml_files; +do + file_tag="$(grep '^version = ' $toml_file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')" + check_tag $current_tag $file_tag $toml_file +done + +lock_file='Cargo.lock' +lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ') +check_tag $current_tag $lock_tag $lock_file + +if [[ "$ret" -eq 0 ]] ; then + echo 'OK' +fi +exit $ret diff --git a/.github/is-latest-release.sh b/.github/scripts/is-latest-release.sh similarity index 90% rename from .github/is-latest-release.sh rename to .github/scripts/is-latest-release.sh index 0c1db61c2..81534a2f7 100644 --- a/.github/is-latest-release.sh +++ b/.github/scripts/is-latest-release.sh @@ -1,14 +1,14 @@ #!/bin/sh -# Checks if the current tag should be the latest (in terms of semver and not of release date). -# Ex: previous tag -> v0.10.1 -# new tag -> v0.8.12 -# The new tag should not be the latest -# So it returns "false", the CI should not run for the release v0.8.2 - -# Used in GHA in publish-docker-latest.yml +# Was used in our CIs to publish the latest docker image. Not used anymore, will be used again when v1 and v2 will be out and we will want to maintain multiple stable versions. # Returns "true" or "false" (as a string) to be used in the `if` in GHA +# Checks if the current tag should be the latest (in terms of semver and not of release date). +# Ex: previous tag -> v2.1.1 +# new tag -> v1.20.3 +# The new tag (v1.20.3) should NOT be the latest +# So it returns "false", the `latest` tag should not be updated for the release v1.20.3 and still need to correspond to v2.1.1 + # GLOBAL GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number] diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 304798d75..298082816 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -5,9 +5,33 @@ on: name: Publish binaries to release jobs: + check-version: + name: Check the version validity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # Check if the tag has the v.. format. + # If yes, it means we are publishing an official release. + # If no, we are releasing a RC, so no need to check the version. + - name: Check tag format + if: github.event_name != 'schedule' + id: check-tag-format + run: | + escaped_tag=$(printf "%q" ${{ github.ref_name }}) + + if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=stable::true + else + echo ::set-output name=stable::false + fi + - name: Check release validity + if: steps.check-tag-format.outputs.stable == 'true' + run: bash .github/scripts/check-release.sh + publish: name: Publish binary for ${{ matrix.os }} runs-on: ${{ matrix.os }} + needs: check-version strategy: fail-fast: false matrix: @@ -41,6 +65,7 @@ jobs: publish-aarch64: name: Publish binary for aarch64 runs-on: ${{ matrix.os }} + needs: check-version continue-on-error: false strategy: fail-fast: false diff --git a/.github/workflows/publish-deb-brew-pkg.yml b/.github/workflows/publish-deb-brew-pkg.yml index 6a5a21287..dbdbdda7e 100644 --- a/.github/workflows/publish-deb-brew-pkg.yml +++ b/.github/workflows/publish-deb-brew-pkg.yml @@ -5,9 +5,18 @@ on: types: [released] jobs: + check-version: + name: Check the version validity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Check release validity + run: bash .github/scripts/check-release.sh + debian: name: Publish debian packagge runs-on: ubuntu-18.04 + needs: check-version steps: - uses: hecrj/setup-rust-action@master with: @@ -30,6 +39,7 @@ jobs: homebrew: name: Bump Homebrew formula runs-on: ubuntu-18.04 + needs: check-version steps: - name: Create PR to Homebrew uses: mislav/bump-homebrew-formula-action@v1 diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml new file mode 100644 index 000000000..0d2e2b60e --- /dev/null +++ b/.github/workflows/publish-docker-images.yml @@ -0,0 +1,71 @@ +--- +on: + schedule: + - cron: '0 4 * * *' # Every day at 4:00am + push: + tags: + - '*' + +name: Publish tagged images to Docker Hub + +jobs: + docker: + runs-on: docker + steps: + - uses: actions/checkout@v2 + + # Check if the tag has the v.. format. If yes, it means we are publishing an official release. + # In this situation, we need to set `output.stable` to create/update the following tags (additionally to the `vX.Y.Z` Docker tag): + # - a `vX.Y` (without patch version) Docker tag + # - a `latest` Docker tag + - name: Check tag format + if: github.event_name != 'schedule' + id: check-tag-format + run: | + escaped_tag=$(printf "%q" ${{ github.ref_name }}) + + if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=stable::true + else + echo ::set-output name=stable::false + fi + + # Check only the validity of the tag for official releases (not for pre-releases or other tags) + - name: Check release validity + if: github.event_name != 'schedule' && steps.check-tag-format.outputs.stable == 'true' + run: bash .github/scripts/check-release.sh + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + if: github.event_name != 'schedule' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: getmeili/meilisearch + # The lastest and `vX.Y` tags are only pushed for the official Meilisearch releases + # See https://github.com/docker/metadata-action#latest-tag + flavor: latest=false + tags: | + type=ref,event=tag + type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }} + type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' }} + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + # We do not push tags for the cron jobs, this is only for test purposes + push: ${{ github.event_name != 'schedule' }} + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/publish-docker-latest.yml b/.github/workflows/publish-docker-latest.yml deleted file mode 100644 index 59cbf9123..000000000 --- a/.github/workflows/publish-docker-latest.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -on: - release: - types: [released] - -name: Publish latest image to Docker Hub - -jobs: - docker-latest: - runs-on: docker - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - platforms: linux/amd64,linux/arm64 - tags: getmeili/meilisearch:latest diff --git a/.github/workflows/publish-docker-tag.yml b/.github/workflows/publish-docker-tag.yml deleted file mode 100644 index eca3d1d25..000000000 --- a/.github/workflows/publish-docker-tag.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -on: - push: - tags: - - '*' - -name: Publish tagged image to Docker Hub - -jobs: - docker-tag: - runs-on: docker - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v3 - with: - images: getmeili/meilisearch - flavor: latest=false - tags: type=ref,event=tag - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ff28f82ca..748c5d690 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -12,6 +12,7 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + RUSTFLAGS: "-D warnings" jobs: tests: @@ -82,7 +83,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly + toolchain: stable override: true components: rustfmt - name: Cache dependencies diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2674c4d1..9b733665c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,7 @@ Remember that there are many ways to contribute other than writing code: writing - [How to Contribute](#how-to-contribute) - [Development Workflow](#development-workflow) - [Git Guidelines](#git-guidelines) +- [Release Process (for internal team only)](#release-process-for-internal-team-only) ## Assumptions @@ -78,6 +79,19 @@ Some notes on GitHub PRs: The draft PRs are recommended when you want to show that you are working on something and make your work visible. - The branch related to the PR must be **up-to-date with `main`** before merging. Fortunately, this project uses [Bors](https://github.com/bors-ng/bors-ng) to automatically enforce this requirement without the PR author having to rebase manually. +## Release Process (for internal team only) + +Meilisearch tools follow the [Semantic Versioning Convention](https://semver.org/). + +### Automation to rebase and Merge the PRs + +This project integrates a bot that helps us manage pull requests merging.
+_[Read more about this](https://github.com/meilisearch/integration-guides/blob/main/resources/bors.md)._ + +### How to Publish a new Release + +The full Meilisearch release process is described in [this guide](https://github.com/meilisearch/core-team/blob/main/resources/meilisearch-release.md). Please follow it carefully before doing any release. +
Thank you again for reading this through, we can not wait to begin to work with you if you made your way through this contributing guide ❤️ diff --git a/Cargo.lock b/Cargo.lock index 81072ecdb..82f83375e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,7 +59,7 @@ dependencies = [ "http", "httparse", "httpdate", - "itoa 1.0.1", + "itoa 1.0.2", "language-tags", "local-channel", "log", @@ -77,8 +77,8 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ - "quote 1.0.17", - "syn 1.0.91", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -188,7 +188,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "itoa 1.0.1", + "itoa 1.0.2", "language-tags", "log", "mime", @@ -211,9 +211,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7525bedf54704abb1d469e88d7e7e9226df73778798a69cea5022d53b2ae91bc" dependencies = [ "actix-router", - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -285,9 +285,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" dependencies = [ "backtrace", ] @@ -318,27 +318,27 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] name = "async-trait" -version = "0.1.53" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] name = "atomic-polyfill" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d862f14e042f75b95236d4ef1bb3d5c170964082d1e1e9c3ce689a2cbee217c" +checksum = "e14bf7b4f565e5e717d7a7a65b2a05c0b8c96e4db636d6f780f03b15108cdd1b" dependencies = [ "critical-section", ] @@ -368,9 +368,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" +checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61" dependencies = [ "addr2line", "cc", @@ -509,9 +509,9 @@ checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" [[package]] name = "bumpalo" -version = "3.9.1" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" [[package]] name = "byte-unit" @@ -525,9 +525,9 @@ dependencies = [ [[package]] name = "bytecount" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72feb31ffc86498dacdbd0fcebb56138e7177a8cc5cea4516031d15ae85a742e" +checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" [[package]] name = "bytemuck" @@ -544,9 +544,9 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562e382481975bc61d11275ac5e62a19abd00b0547d99516a415336f183dcd0e" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -643,12 +643,33 @@ dependencies = [ ] [[package]] -name = "character_converter" -version = "1.0.0" +name = "charabia" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e48477ece09d6a21c033cb604968524a37782532727055d6f6faafac1781e5c" +checksum = "4a26a3df4d9c9231eb1e757fe6b1c66c471e0c2cd5410265e7c3109a726663c4" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "lindera", + "lindera-core", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + +[[package]] +name = "character_converter" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7064c6e919124b6541c52fef59d88c3c3eabdf4bc97c13b14551df775aead02" dependencies = [ "bincode", + "fst", + "once_cell", ] [[package]] @@ -659,16 +680,16 @@ checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" [[package]] name = "clap" -version = "3.1.8" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c" +checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" dependencies = [ "atty", "bitflags", "clap_derive", + "clap_lex", "indexmap", "lazy_static", - "os_str_bytes", "strsim", "termcolor", "textwrap", @@ -676,15 +697,24 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.1.7" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" +checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", +] + +[[package]] +name = "clap_lex" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" +dependencies = [ + "os_str_bytes", ] [[package]] @@ -693,9 +723,9 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df715824eb382e34b7afb7463b0247bf41538aeba731fba05241ecdb5dc3747" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -723,9 +753,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cortex-m" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ff967e867ca14eba0c34ac25cd71ea98c678e741e3915d923999bb2fe7c826" +checksum = "cd20d4ac4aa86f4f75f239d59e542ef67de87cce2c282818dc6e84155d3ea126" dependencies = [ "bare-metal 0.2.5", "bitfield", @@ -768,9 +798,9 @@ dependencies = [ [[package]] name = "critical-section" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1e89b93912c97878305b70ef6b011bfc74622e7b79a9d4a0676c7663496bcd" +checksum = "95da181745b56d4bd339530ec393508910c909c784e8962d15d722bacf0bcbcd" dependencies = [ "bare-metal 1.0.0", "cfg-if 1.0.0", @@ -880,9 +910,9 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -892,10 +922,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.37", - "quote 1.0.17", + "proc-macro2 1.0.39", + "quote 1.0.18", "rustc_version 0.4.0", - "syn 1.0.91", + "syn 1.0.96", ] [[package]] @@ -918,6 +948,7 @@ checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -1051,9 +1082,9 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -1080,9 +1111,9 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" +checksum = "c0408e2626025178a6a7f7ffc05a25bc47103229f19c113755de7bf63816290c" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1092,8 +1123,8 @@ dependencies = [ [[package]] name = "filter-parser" -version = "0.26.6" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.6#955aff48853c6032c75970dc710929f08b4f711a" +version = "0.31.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8" dependencies = [ "nom", "nom_locate", @@ -1101,26 +1132,24 @@ dependencies = [ [[package]] name = "firestorm" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d3d6188b8804df28032815ea256b6955c9625c24da7525f387a7af02fbb8f01" +checksum = "2c5f6c2c942da57e2aaaa84b8a521489486f14e75e7fa91dab70aba913975f98" [[package]] name = "flate2" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ - "cfg-if 1.0.0", "crc32fast", - "libc", "miniz_oxide", ] [[package]] name = "flatten-serde-json" -version = "0.26.6" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.6#955aff48853c6032c75970dc710929f08b4f711a" +version = "0.31.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8" dependencies = [ "serde_json", ] @@ -1152,9 +1181,9 @@ dependencies = [ [[package]] name = "fragile" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d758e60b45e8d749c89c1b389ad8aee550f86aa12e2b9298b546dda7a82ab1" +checksum = "85dcb89d2b10c5f6133de2efd8c11959ce9dbb46a2f7a4cab208c4eeda6ce1ab" [[package]] name = "fs_extra" @@ -1222,9 +1251,9 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -1300,9 +1329,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -1313,9 +1342,9 @@ checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" [[package]] name = "git2" -version = "0.14.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3826a6e0e2215d7a41c2bfc7c9244123969273f3476b939a226aac0ab56e9e3c" +checksum = "d0155506aab710a86160ddb504a480d2964d7ab5b9e62419be69e0032bc5931c" dependencies = [ "bitflags", "libc", @@ -1332,9 +1361,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "grenad" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d69e46e7b225459e2e0272707d167d7dcaaac89307a848326df6b30ec432151" +checksum = "3e8454188b8caee0627ff58636048963b6abd07e5862b4c9a8f9cfd349d50c26" dependencies = [ "bytemuck", "byteorder", @@ -1387,13 +1416,14 @@ checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" [[package]] name = "heapless" -version = "0.7.10" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d076121838e03f862871315477528debffdb7462fb229216ecef91b1a3eb31eb" +checksum = "8a08e755adbc0ad283725b29f4a4883deee15336f372d5f61fae59efec40f983" dependencies = [ "atomic-polyfill", "hash32", - "spin 0.9.2", + "rustc_version 0.4.0", + "spin 0.9.3", "stable_deref_trait", ] @@ -1453,21 +1483,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "http" -version = "0.2.6" +name = "hmac" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa 1.0.1", + "itoa 1.0.2", ] [[package]] name = "http-body" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", @@ -1476,9 +1515,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" @@ -1494,9 +1533,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.18" +version = "0.14.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +checksum = "42dc3c131584288d375f2d07f822b0cb012d8c6fb899a5b9fdb3cb7eb9b6004f" dependencies = [ "bytes", "futures-channel", @@ -1507,7 +1546,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.1", + "itoa 1.0.2", "pin-project-lite", "socket2", "tokio", @@ -1542,9 +1581,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" dependencies = [ "autocfg", "hashbrown 0.11.2", @@ -1562,9 +1601,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "itertools" @@ -1583,9 +1622,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "jieba-rs" @@ -1613,26 +1652,26 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" dependencies = [ "wasm-bindgen", ] [[package]] name = "json-depth-checker" -version = "0.26.6" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.6#955aff48853c6032c75970dc710929f08b4f711a" +version = "0.31.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8" dependencies = [ "serde_json", ] [[package]] name = "jsonwebtoken" -version = "8.0.1" +version = "8.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "012bb02250fdd38faa5feee63235f7a459974440b9b57593822414c31f92839e" +checksum = "cc9051c17f81bae79440afa041b3a278e1de71bfb96d32454b477fd4703ccb6f" dependencies = [ "base64", "pem", @@ -1665,15 +1704,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec647867e2bf0772e28c8bcde4f0d19a9216916e890543b5a03ed8ef27b8f259" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libgit2-sys" -version = "0.13.2+1.4.2" +version = "0.13.4+1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a42de9a51a5c12e00fc0e4ca6bc2ea43582fc6418488e8f615e905d886f258b" +checksum = "d0fa6563431ede25f5cc7f6d803c6afbc1c5d3ad3d4925d12c882bf2b526f5d1" dependencies = [ "cc", "libc", @@ -1689,9 +1728,9 @@ checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db" [[package]] name = "libz-sys" -version = "1.1.5" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" dependencies = [ "cc", "libc", @@ -1701,9 +1740,9 @@ dependencies = [ [[package]] name = "lindera" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dea10df226936ff54f16d3922500e08ef4be2ba7c0070bec9ad4a1474316111" +checksum = "7d1c5db4b1d12637aa316dc1adb215f78fe79025080af750942516c5ff17d1a0" dependencies = [ "anyhow", "bincode", @@ -1723,9 +1762,9 @@ dependencies = [ [[package]] name = "lindera-cc-cedict-builder" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4392785248c3d8755c6fae9d0086d27ad7a1d6810155a2494fe5206e2021f471" +checksum = "73a3509fb497340571d49feddb57e1db2ce5248c4d449f2548d0ee8cb745eb1e" dependencies = [ "anyhow", "bincode", @@ -1743,9 +1782,9 @@ dependencies = [ [[package]] name = "lindera-core" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af63a4484334d4b83277621f1ba62fb83472858cc37fb4ab2181a4c19eebcb38" +checksum = "5d20d1b2c085393aed58625d741beca69410e1143fc35bc67ebc35c9885f9f74" dependencies = [ "anyhow", "bincode", @@ -1759,9 +1798,9 @@ dependencies = [ [[package]] name = "lindera-decompress" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "817ee62bc8973ec2457805df83796c59f074e49a4a0ee9baffe2663fe157f54a" +checksum = "b96b8050cded13927a99bcb8cbb0987f89fc8f35429fc153b4bc05ddc7a53a44" dependencies = [ "anyhow", "lzma-rs", @@ -1770,9 +1809,9 @@ dependencies = [ [[package]] name = "lindera-dictionary" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd57501ee44a6aba0431d043c7926347e29883a79d8fc3955b8837e4ad1fee3c" +checksum = "5abe3dddc22303402957edb4472ab0c996e0d93b3b00643de3bee8b28c2f9297" dependencies = [ "anyhow", "bincode", @@ -1782,9 +1821,9 @@ dependencies = [ [[package]] name = "lindera-ipadic" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade3bd3faa5f0db629c26264663e901dee5f46221eb04c2c7b592bd7485d44f9" +checksum = "b8f4c111f6ad9eb9e015d02061af2ed36fc0255f29359294415c7c2f1ea5b5b6" dependencies = [ "bincode", "byteorder", @@ -1799,9 +1838,9 @@ dependencies = [ [[package]] name = "lindera-ipadic-builder" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee61f8dd6566738c5fd0ee9b1c11212ffc2d1f97af69c08a02cbb5c49995250a" +checksum = "a2b9893f22a4a7511ac70ff7d96cda9b8d7259b7d7121784183c73bc593ce6e7" dependencies = [ "anyhow", "bincode", @@ -1819,9 +1858,9 @@ dependencies = [ [[package]] name = "lindera-ko-dic-builder" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01f05950d9adc7aa42aa8b16be1616f9625576c867179ac29372714eaed6993d" +checksum = "14282600ebfe7ab6fd4f3042143024ff9d74c09d58fd983d0c587839cf940d4a" dependencies = [ "anyhow", "bincode", @@ -1839,9 +1878,9 @@ dependencies = [ [[package]] name = "lindera-unidic-builder" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3836c1278b8309ebf209c67bc7a935f4ce7c9246a578b250540398806a40b81d" +checksum = "b20825d46c95854e47c532c3e548dfec07c8f187c1ed89383cb6c35790338088" dependencies = [ "anyhow", "bincode", @@ -1869,9 +1908,9 @@ dependencies = [ [[package]] name = "local-channel" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6246c68cf195087205a0512559c97e15eaf95198bf0e206d662092cdcb03fe9f" +checksum = "7f303ec0e94c6c54447f84f3b0ef7af769858a9c4ef56ef2a986d3dcd4c3fc9c" dependencies = [ "futures-core", "futures-sink", @@ -1881,9 +1920,9 @@ dependencies = [ [[package]] name = "local-waker" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "902eb695eb0591864543cbfbf6d742510642a605a61fc5e97fe6ceb5a30ac4fb" +checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" [[package]] name = "lock_api" @@ -1897,9 +1936,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", ] @@ -1921,9 +1960,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10a9062912d7952c5588cc474795e0b9ee008e7e6781127945b85413d4b99d81" dependencies = [ "log", - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -1936,6 +1975,18 @@ dependencies = [ "crc", ] +[[package]] +name = "manifest-dir-macros" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f08150cf2bab1fc47c2196f4f41173a27fcd0f684165e5458c0046b53a472e2f" +dependencies = [ + "once_cell", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", +] + [[package]] name = "maplit" version = "1.0.2" @@ -1950,10 +2001,11 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "meilisearch-auth" -version = "0.27.2" +version = "0.28.0" dependencies = [ "enum-iterator", - "meilisearch-error", + "hmac", + "meilisearch-types", "milli", "rand", "serde", @@ -1961,22 +2013,12 @@ dependencies = [ "sha2", "thiserror", "time 0.3.9", -] - -[[package]] -name = "meilisearch-error" -version = "0.27.2" -dependencies = [ - "actix-web", - "proptest", - "proptest-derive", - "serde", - "serde_json", + "uuid 1.1.2", ] [[package]] name = "meilisearch-http" -version = "0.27.2" +version = "0.28.0" dependencies = [ "actix-cors", "actix-rt", @@ -2004,16 +2046,16 @@ dependencies = [ "itertools", "jsonwebtoken", "log", + "manifest-dir-macros", "maplit", "meilisearch-auth", - "meilisearch-error", "meilisearch-lib", + "meilisearch-types", "mime", "num_cpus", "obkv", "once_cell", "parking_lot", - "paste", "pin-project-lite", "platform-dirs", "rand", @@ -2024,8 +2066,8 @@ dependencies = [ "rustls-pemfile", "segment", "serde", + "serde-cs", "serde_json", - "serde_url_params", "sha-1", "sha2", "siphasher", @@ -2040,15 +2082,16 @@ dependencies = [ "tokio", "tokio-stream", "urlencoding", - "uuid", + "uuid 1.1.2", "vergen", "walkdir", + "yaup", "zip", ] [[package]] name = "meilisearch-lib" -version = "0.27.2" +version = "0.28.0" dependencies = [ "actix-rt", "actix-web", @@ -2074,7 +2117,7 @@ dependencies = [ "lazy_static", "log", "meilisearch-auth", - "meilisearch-error", + "meilisearch-types", "milli", "mime", "mockall", @@ -2091,6 +2134,7 @@ dependencies = [ "rayon", "regex", "reqwest", + "roaring", "rustls", "serde", "serde_json", @@ -2102,40 +2146,33 @@ dependencies = [ "thiserror", "time 0.3.9", "tokio", - "uuid", + "uuid 1.1.2", "walkdir", "whoami", ] [[package]] -name = "meilisearch-tokenizer" -version = "0.2.9" -source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.9#1dfc8ad9f5b338c39c3bc5fd5b2d0c1328314ddc" +name = "meilisearch-types" +version = "0.28.0" dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "lindera", - "lindera-core", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", + "actix-web", + "proptest", + "proptest-derive", + "serde", + "serde_json", ] [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057a3db23999c867821a7a59feb06a578fcb03685e983dff90daf9e7d24ac08f" +checksum = "d5172b50c23043ff43dd53e51392f36519d9b35a8f3a410d30ece5d1aedd58ae" dependencies = [ "libc", ] @@ -2151,13 +2188,14 @@ dependencies = [ [[package]] name = "milli" -version = "0.26.6" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.26.6#955aff48853c6032c75970dc710929f08b4f711a" +version = "0.31.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8" dependencies = [ "bimap", "bincode", "bstr", "byteorder", + "charabia", "concat-arrays", "crossbeam-channel", "csv", @@ -2174,7 +2212,6 @@ dependencies = [ "levenshtein_automata", "log", "logging_timer", - "meilisearch-tokenizer", "memmap2", "obkv", "once_cell", @@ -2189,8 +2226,9 @@ dependencies = [ "smallvec", "smartstring", "tempfile", + "thiserror", "time 0.3.9", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -2217,42 +2255,30 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" dependencies = [ "adler", - "autocfg", ] [[package]] name = "mio" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "windows-sys", ] [[package]] name = "mockall" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d4d70639a72f972725db16350db56da68266ca368b2a1fe26724a903ad3d6b8" +checksum = "5641e476bbaf592a3939a7485fa079f427b4db21407d5ebfd5bba4e07a1f6f4c" dependencies = [ "cfg-if 1.0.0", "downcast", @@ -2265,14 +2291,14 @@ dependencies = [ [[package]] name = "mockall_derive" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79ef208208a0dea3f72221e26e904cdc6db2e481d9ade89081ddd494f1dbaa6b" +checksum = "262d56735932ee0240d515656e5a7667af3af2a5b0af4da558c4cff2b2aeb0c7" dependencies = [ "cfg-if 1.0.0", - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -2293,7 +2319,7 @@ checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae" [[package]] name = "nelson" version = "0.1.0" -source = "git+https://github.com/MarinPostma/nelson.git?rev=675f13885548fb415ead8fbb447e9e6d9314000a#675f13885548fb415ead8fbb447e9e6d9314000a" +source = "git+https://github.com/meilisearch/nelson.git?rev=675f13885548fb415ead8fbb447e9e6d9314000a#675f13885548fb415ead8fbb447e9e6d9314000a" [[package]] name = "nom" @@ -2344,9 +2370,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -2354,9 +2380,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", "libm", @@ -2374,18 +2400,18 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" dependencies = [ "libc", ] [[package]] name = "object" -version = "0.27.1" +version = "0.28.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" dependencies = [ "memchr", ] @@ -2398,9 +2424,9 @@ checksum = "f69e48cd7c8e5bb52a1da1287fdbfd877c32673176583ce664cd63b201aba385" [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "ordered-float" @@ -2413,12 +2439,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.0.0" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" -dependencies = [ - "memchr", -] +checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" [[package]] name = "page_size" @@ -2432,9 +2455,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", "parking_lot_core", @@ -2442,9 +2465,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if 1.0.0", "libc", @@ -2491,7 +2514,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "permissive-json-pointer" -version = "0.2.0" +version = "0.28.0" dependencies = [ "big_s", "serde_json", @@ -2537,9 +2560,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -2605,9 +2628,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", "version_check", ] @@ -2617,8 +2640,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", + "proc-macro2 1.0.39", + "quote 1.0.18", "version_check", ] @@ -2633,11 +2656,11 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.37" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" dependencies = [ - "unicode-xid 0.2.2", + "unicode-ident", ] [[package]] @@ -2683,15 +2706,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" -[[package]] -name = "quickcheck" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" -dependencies = [ - "rand", -] - [[package]] name = "quote" version = "0.6.13" @@ -2703,11 +2717,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ - "proc-macro2 1.0.37", + "proc-macro2 1.0.39", ] [[package]] @@ -2751,9 +2765,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.1" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ "autocfg", "crossbeam-deque", @@ -2763,14 +2777,13 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" dependencies = [ "crossbeam-channel", "crossbeam-deque", "crossbeam-utils 0.8.8", - "lazy_static", "num_cpus", ] @@ -2796,9 +2809,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" dependencies = [ "aho-corasick", "memchr", @@ -2813,9 +2826,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "remove_dir_all" @@ -2866,9 +2879,9 @@ dependencies = [ [[package]] name = "retain_mut" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c31b5c4033f8fdde8700e4657be2c497e7288f01515be52168c631e2e4d4086" +checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" [[package]] name = "ring" @@ -2950,14 +2963,14 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.7", + "semver 1.0.9", ] [[package]] name = "rustls" -version = "0.20.4" +version = "0.20.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbfeb8d0ddb84706bc597a5574ab8912817c52a397f819e5b614e2265206921" +checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" dependencies = [ "log", "ring", @@ -2994,9 +3007,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "same-file" @@ -3048,9 +3061,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" [[package]] name = "semver-parser" @@ -3060,46 +3073,45 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] [[package]] -name = "serde_derive" -version = "1.0.136" +name = "serde-cs" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "8202c9f3f58762d274952790ff8a1f1f625b5664f75e5dc1952c8dcacc64a925" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +dependencies = [ + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "indexmap", - "itoa 1.0.1", + "itoa 1.0.2", "ryu", "serde", ] -[[package]] -name = "serde_url_params" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c43307d0640738af32fe8d01e47119bc0fc8a686be470a44a586caff76dfb34" -dependencies = [ - "serde", - "url", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3107,7 +3119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.1", + "itoa 1.0.2", "ryu", "serde", ] @@ -3145,9 +3157,9 @@ dependencies = [ [[package]] name = "simple_asn1" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a762b1c38b9b990c694b9c2f8abe3372ce6a9ceaae6bca39cfc46e054f45745" +checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" dependencies = [ "num-bigint", "num-traits", @@ -3218,9 +3230,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "spin" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" +checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d" dependencies = [ "lock_api", ] @@ -3254,6 +3266,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "syn" version = "0.15.44" @@ -3267,13 +3285,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.91" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" +checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "unicode-xid 0.2.2", + "proc-macro2 1.0.39", + "quote 1.0.18", + "unicode-ident", ] [[package]] @@ -3291,17 +3309,17 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", - "unicode-xid 0.2.2", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", + "unicode-xid 0.2.3", ] [[package]] name = "sysinfo" -version = "0.23.8" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad04c584871b8dceb769a20b94e26a357a870c999b7246dcd4cb233d927547e3" +checksum = "3977ec2e0520829be45c8a2df70db2bf364714d8a748316a10c3c35d4d2b01c9" dependencies = [ "cfg-if 1.0.0", "core-foundation-sys", @@ -3360,22 +3378,22 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] @@ -3415,10 +3433,9 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ - "itoa 1.0.1", + "itoa 1.0.2", "libc", "num_threads", - "quickcheck", "serde", "time-macros", ] @@ -3431,9 +3448,9 @@ checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -3446,9 +3463,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" dependencies = [ "bytes", "libc", @@ -3466,20 +3483,20 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", ] [[package]] name = "tokio-rustls" -version = "0.23.3" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ "rustls", "tokio", @@ -3488,9 +3505,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" dependencies = [ "futures-core", "pin-project-lite", @@ -3499,9 +3516,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ "bytes", "futures-core", @@ -3513,9 +3530,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] @@ -3528,35 +3545,23 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.32" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ "cfg-if 1.0.0", "log", "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" -dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", -] - [[package]] name = "tracing-core" -version = "0.1.24" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90442985ee2f57c9e1b548ee72ae842f4a9a20e3f417cc38dbc5dc684d9bb4ee" +checksum = "7709595b8878a4965ce5e87ebf880a7d39c9afc6837721b21a5a816a8117d921" dependencies = [ - "lazy_static", + "once_cell", ] [[package]] @@ -3582,9 +3587,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" [[package]] name = "unicode-normalization" @@ -3609,9 +3620,9 @@ checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "untrusted" @@ -3664,6 +3675,15 @@ name = "uuid" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" dependencies = [ "getrandom", "serde", @@ -3683,9 +3703,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "7.0.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4db743914c971db162f35bf46601c5a63ec4452e61461937b4c1ab817a60c12e" +checksum = "b1f44ef1afcf5979e34748c12595f9589f3dc4e34abf156fb6d95f9b835568dc" dependencies = [ "anyhow", "cfg-if 1.0.0", @@ -3762,9 +3782,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3772,24 +3792,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3799,38 +3819,38 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ - "quote 1.0.17", + "quote 1.0.18", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ - "proc-macro2 1.0.37", - "quote 1.0.17", - "syn 1.0.91", + "proc-macro2 1.0.39", + "quote 1.0.18", + "syn 1.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" [[package]] name = "web-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" dependencies = [ "js-sys", "wasm-bindgen", @@ -3907,9 +3927,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -3920,33 +3940,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "winreg" @@ -3959,9 +3979,9 @@ dependencies = [ [[package]] name = "xattr" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" dependencies = [ "libc", ] @@ -3972,6 +3992,16 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d12cb7a57bbf2ab670ed9545bae3648048547f9039279a89ce000208e585c1" +[[package]] +name = "yaup" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bc9ef6963f7e857050aabf31ebc44184f278bcfec4c3671552c1a916b152b45" +dependencies = [ + "serde", + "url", +] + [[package]] name = "zerocopy" version = "0.3.0" @@ -3988,8 +4018,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ - "proc-macro2 1.0.37", - "syn 1.0.91", + "proc-macro2 1.0.39", + "syn 1.0.96", "synstructure", ] diff --git a/Cargo.toml b/Cargo.toml index 03f4f5597..99ec43528 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,14 @@ resolver = "2" members = [ "meilisearch-http", - "meilisearch-error", + "meilisearch-types", "meilisearch-lib", "meilisearch-auth", "permissive-json-pointer", ] + +[profile.dev.package.flate2] +opt-level = 3 + +[profile.dev.package.milli] +opt-level = 3 diff --git a/README.md b/README.md index 9efb5a937..d882ba0de 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ meilisearch #### Docker ```bash -docker run -p 7700:7700 -v "$(pwd)/data.ms:/data.ms" getmeili/meilisearch +docker run -p 7700:7700 -v "$(pwd)/meili_data:/meili_data" getmeili/meilisearch ``` #### Announcing a cloud-hosted Meilisearch @@ -109,7 +109,7 @@ cargo run --release Let's create an index! If you need a sample dataset, use [this movie database](https://www.notion.so/meilisearch/A-movies-dataset-to-test-Meili-1cbf7c9cfa4247249c40edfa22d7ca87#b5ae399b81834705ba5420ac70358a65). You can also find it in the `datasets/` directory. ```bash -curl -L 'https://bit.ly/2PAcw9l' -o movies.json +curl -L https://docs.meilisearch.com/movies.json -o movies.json ``` Now, you're ready to index some data. diff --git a/SECURITY.md b/SECURITY.md index 63bc15a40..5de4af60d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ Meilisearch takes the security of our software products and services seriously. If you believe you have found a security vulnerability in any Meilisearch-owned repository, please report it to us as described below. -## Suported versions +## Supported versions As long as we are pre-v1.0, only the latest version of Meilisearch will be supported with security updates. diff --git a/bors.toml b/bors.toml index d24e6c09b..b357e8d61 100644 --- a/bors.toml +++ b/bors.toml @@ -2,7 +2,7 @@ status = [ 'Tests on ubuntu-18.04', 'Tests on macos-latest', 'Tests on windows-latest', - # 'Run Clippy', + 'Run Clippy', 'Run Rustfmt', 'Run tests in debug', ] diff --git a/download-latest.sh b/download-latest.sh index 6fa714c55..d1cfdd127 100644 --- a/download-latest.sh +++ b/download-latest.sh @@ -67,8 +67,8 @@ semverLT() { return 1 } -# Get a token from https://github.com/settings/tokens to increasae rate limit (from 60 to 5000), make sure the token scope is set to 'public_repo' -# Create GITHUB_PAT enviroment variable once you aquired the token to start using it +# Get a token from https://github.com/settings/tokens to increase rate limit (from 60 to 5000), make sure the token scope is set to 'public_repo' +# Create GITHUB_PAT environment variable once you acquired the token to start using it # Returns the tag of the latest stable release (in terms of semver and not of release date) get_latest() { temp_file='temp_file' # temp_file needed because the grep would start before the download is over @@ -89,7 +89,7 @@ get_latest() { latest='' current_tag='' for release_info in $releases; do - if [ $i -eq 0 ]; then # Cheking tag_name + if [ $i -eq 0 ]; then # Checking tag_name if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release current_tag=$release_info else diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index 27586db98..3ba5408e8 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -1,15 +1,17 @@ [package] name = "meilisearch-auth" -version = "0.27.2" +version = "0.28.0" edition = "2021" [dependencies] enum-iterator = "0.7.0" -meilisearch-error = { path = "../meilisearch-error" } -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.26.6" } +hmac = "0.12.1" +meilisearch-types = { path = "../meilisearch-types" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.31.1" } rand = "0.8.4" serde = { version = "1.0.136", features = ["derive"] } serde_json = { version = "1.0.79", features = ["preserve_order"] } sha2 = "0.10.2" thiserror = "1.0.30" time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } +uuid = { version = "1.1.2", features = ["serde", "v4"] } diff --git a/meilisearch-auth/src/action.rs b/meilisearch-auth/src/action.rs index 7ffe9b908..fab5263ec 100644 --- a/meilisearch-auth/src/action.rs +++ b/meilisearch-auth/src/action.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; #[repr(u8)] pub enum Action { #[serde(rename = "*")] - All = 0, + All = actions::ALL, #[serde(rename = "search")] Search = actions::SEARCH, #[serde(rename = "documents.add")] @@ -32,17 +32,23 @@ pub enum Action { StatsGet = actions::STATS_GET, #[serde(rename = "dumps.create")] DumpsCreate = actions::DUMPS_CREATE, - #[serde(rename = "dumps.get")] - DumpsGet = actions::DUMPS_GET, #[serde(rename = "version")] Version = actions::VERSION, + #[serde(rename = "keys.create")] + KeysAdd = actions::KEYS_CREATE, + #[serde(rename = "keys.get")] + KeysGet = actions::KEYS_GET, + #[serde(rename = "keys.update")] + KeysUpdate = actions::KEYS_UPDATE, + #[serde(rename = "keys.delete")] + KeysDelete = actions::KEYS_DELETE, } impl Action { pub fn from_repr(repr: u8) -> Option { use actions::*; match repr { - 0 => Some(Self::All), + ALL => Some(Self::All), SEARCH => Some(Self::Search), DOCUMENTS_ADD => Some(Self::DocumentsAdd), DOCUMENTS_GET => Some(Self::DocumentsGet), @@ -56,8 +62,11 @@ impl Action { SETTINGS_UPDATE => Some(Self::SettingsUpdate), STATS_GET => Some(Self::StatsGet), DUMPS_CREATE => Some(Self::DumpsCreate), - DUMPS_GET => Some(Self::DumpsGet), VERSION => Some(Self::Version), + KEYS_CREATE => Some(Self::KeysAdd), + KEYS_GET => Some(Self::KeysGet), + KEYS_UPDATE => Some(Self::KeysUpdate), + KEYS_DELETE => Some(Self::KeysDelete), _otherwise => None, } } @@ -65,7 +74,7 @@ impl Action { pub fn repr(&self) -> u8 { use actions::*; match self { - Self::All => 0, + Self::All => ALL, Self::Search => SEARCH, Self::DocumentsAdd => DOCUMENTS_ADD, Self::DocumentsGet => DOCUMENTS_GET, @@ -79,13 +88,17 @@ impl Action { Self::SettingsUpdate => SETTINGS_UPDATE, Self::StatsGet => STATS_GET, Self::DumpsCreate => DUMPS_CREATE, - Self::DumpsGet => DUMPS_GET, Self::Version => VERSION, + Self::KeysAdd => KEYS_CREATE, + Self::KeysGet => KEYS_GET, + Self::KeysUpdate => KEYS_UPDATE, + Self::KeysDelete => KEYS_DELETE, } } } pub mod actions { + pub(crate) const ALL: u8 = 0; pub const SEARCH: u8 = 1; pub const DOCUMENTS_ADD: u8 = 2; pub const DOCUMENTS_GET: u8 = 3; @@ -99,6 +112,9 @@ pub mod actions { pub const SETTINGS_UPDATE: u8 = 11; pub const STATS_GET: u8 = 12; pub const DUMPS_CREATE: u8 = 13; - pub const DUMPS_GET: u8 = 14; pub const VERSION: u8 = 15; + pub const KEYS_CREATE: u8 = 16; + pub const KEYS_GET: u8 = 17; + pub const KEYS_UPDATE: u8 = 18; + pub const KEYS_DELETE: u8 = 19; } diff --git a/meilisearch-auth/src/dump.rs b/meilisearch-auth/src/dump.rs index 77a4aa5ca..7e607e574 100644 --- a/meilisearch-auth/src/dump.rs +++ b/meilisearch-auth/src/dump.rs @@ -1,5 +1,6 @@ +use serde_json::Deserializer; + use std::fs::File; -use std::io::BufRead; use std::io::BufReader; use std::io::Write; use std::path::Path; @@ -36,10 +37,9 @@ impl AuthController { return Ok(()); } - let mut reader = BufReader::new(File::open(&keys_file_path)?).lines(); - while let Some(key) = reader.next().transpose()? { - let key = serde_json::from_str(&key)?; - store.put_api_key(key)?; + let reader = BufReader::new(File::open(&keys_file_path)?); + for key in Deserializer::from_reader(reader).into_iter() { + store.put_api_key(key?)?; } Ok(()) diff --git a/meilisearch-auth/src/error.rs b/meilisearch-auth/src/error.rs index 8a87eda27..bb96be789 100644 --- a/meilisearch-auth/src/error.rs +++ b/meilisearch-auth/src/error.rs @@ -1,7 +1,7 @@ use std::error::Error; -use meilisearch_error::ErrorCode; -use meilisearch_error::{internal_error, Code}; +use meilisearch_types::error::{Code, ErrorCode}; +use meilisearch_types::internal_error; use serde_json::Value; pub type Result = std::result::Result; @@ -18,8 +18,18 @@ pub enum AuthControllerError { InvalidApiKeyExpiresAt(Value), #[error("`description` field value `{0}` is invalid. It should be a string or specified as a null value.")] InvalidApiKeyDescription(Value), + #[error( + "`name` field value `{0}` is invalid. It should be a string or specified as a null value." + )] + InvalidApiKeyName(Value), + #[error("`uid` field value `{0}` is invalid. It should be a valid UUID v4 string or omitted.")] + InvalidApiKeyUid(Value), #[error("API key `{0}` not found.")] ApiKeyNotFound(String), + #[error("`uid` field value `{0}` is already an existing API key.")] + ApiKeyAlreadyExists(String), + #[error("The `{0}` field cannot be modified for the given resource.")] + ImmutableField(String), #[error("Internal error: {0}")] Internal(Box), } @@ -39,7 +49,11 @@ impl ErrorCode for AuthControllerError { Self::InvalidApiKeyIndexes(_) => Code::InvalidApiKeyIndexes, Self::InvalidApiKeyExpiresAt(_) => Code::InvalidApiKeyExpiresAt, Self::InvalidApiKeyDescription(_) => Code::InvalidApiKeyDescription, + Self::InvalidApiKeyName(_) => Code::InvalidApiKeyName, Self::ApiKeyNotFound(_) => Code::ApiKeyNotFound, + Self::InvalidApiKeyUid(_) => Code::InvalidApiKeyUid, + Self::ApiKeyAlreadyExists(_) => Code::ApiKeyAlreadyExists, + Self::ImmutableField(_) => Code::ImmutableField, Self::Internal(_) => Code::Internal, } } diff --git a/meilisearch-auth/src/key.rs b/meilisearch-auth/src/key.rs index 1b06f34be..eb72aaa72 100644 --- a/meilisearch-auth/src/key.rs +++ b/meilisearch-auth/src/key.rs @@ -1,20 +1,25 @@ use crate::action::Action; use crate::error::{AuthControllerError, Result}; -use crate::store::{KeyId, KEY_ID_LENGTH}; -use rand::Rng; +use crate::store::KeyId; + +use meilisearch_types::index_uid::IndexUid; +use meilisearch_types::star_or::StarOr; use serde::{Deserialize, Serialize}; use serde_json::{from_value, Value}; use time::format_description::well_known::Rfc3339; use time::macros::{format_description, time}; use time::{Date, OffsetDateTime, PrimitiveDateTime}; +use uuid::Uuid; #[derive(Debug, Deserialize, Serialize)] pub struct Key { #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, - pub id: KeyId, + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + pub uid: KeyId, pub actions: Vec, - pub indexes: Vec, + pub indexes: Vec>, #[serde(with = "time::serde::rfc3339::option")] pub expires_at: Option, #[serde(with = "time::serde::rfc3339")] @@ -25,16 +30,27 @@ pub struct Key { impl Key { pub fn create_from_value(value: Value) -> Result { - let description = match value.get("description") { - Some(Value::Null) => None, - Some(des) => Some( - from_value(des.clone()) - .map_err(|_| AuthControllerError::InvalidApiKeyDescription(des.clone()))?, - ), - None => None, + let name = match value.get("name") { + None | Some(Value::Null) => None, + Some(des) => from_value(des.clone()) + .map(Some) + .map_err(|_| AuthControllerError::InvalidApiKeyName(des.clone()))?, }; - let id = generate_id(); + let description = match value.get("description") { + None | Some(Value::Null) => None, + Some(des) => from_value(des.clone()) + .map(Some) + .map_err(|_| AuthControllerError::InvalidApiKeyDescription(des.clone()))?, + }; + + let uid = value.get("uid").map_or_else( + || Ok(Uuid::new_v4()), + |uid| { + from_value(uid.clone()) + .map_err(|_| AuthControllerError::InvalidApiKeyUid(uid.clone())) + }, + )?; let actions = value .get("actions") @@ -61,8 +77,9 @@ impl Key { let updated_at = created_at; Ok(Self { + name, description, - id, + uid, actions, indexes, expires_at, @@ -78,20 +95,34 @@ impl Key { self.description = des?; } - if let Some(act) = value.get("actions") { - let act = from_value(act.clone()) - .map_err(|_| AuthControllerError::InvalidApiKeyActions(act.clone())); - self.actions = act?; + if let Some(des) = value.get("name") { + let des = from_value(des.clone()) + .map_err(|_| AuthControllerError::InvalidApiKeyName(des.clone())); + self.name = des?; } - if let Some(ind) = value.get("indexes") { - let ind = from_value(ind.clone()) - .map_err(|_| AuthControllerError::InvalidApiKeyIndexes(ind.clone())); - self.indexes = ind?; + if value.get("uid").is_some() { + return Err(AuthControllerError::ImmutableField("uid".to_string())); } - if let Some(exp) = value.get("expiresAt") { - self.expires_at = parse_expiration_date(exp)?; + if value.get("actions").is_some() { + return Err(AuthControllerError::ImmutableField("actions".to_string())); + } + + if value.get("indexes").is_some() { + return Err(AuthControllerError::ImmutableField("indexes".to_string())); + } + + if value.get("expiresAt").is_some() { + return Err(AuthControllerError::ImmutableField("expiresAt".to_string())); + } + + if value.get("createdAt").is_some() { + return Err(AuthControllerError::ImmutableField("createdAt".to_string())); + } + + if value.get("updatedAt").is_some() { + return Err(AuthControllerError::ImmutableField("updatedAt".to_string())); } self.updated_at = OffsetDateTime::now_utc(); @@ -101,11 +132,13 @@ impl Key { pub(crate) fn default_admin() -> Self { let now = OffsetDateTime::now_utc(); + let uid = Uuid::new_v4(); Self { - description: Some("Default Admin API Key (Use it for all other operations. Caution! Do not use it on a public frontend)".to_string()), - id: generate_id(), + name: Some("Default Admin API Key".to_string()), + description: Some("Use it for anything that is not a search operation. Caution! Do not expose it on a public frontend".to_string()), + uid, actions: vec![Action::All], - indexes: vec!["*".to_string()], + indexes: vec![StarOr::Star], expires_at: None, created_at: now, updated_at: now, @@ -114,13 +147,13 @@ impl Key { pub(crate) fn default_search() -> Self { let now = OffsetDateTime::now_utc(); + let uid = Uuid::new_v4(); Self { - description: Some( - "Default Search API Key (Use it to search from the frontend)".to_string(), - ), - id: generate_id(), + name: Some("Default Search API Key".to_string()), + description: Some("Use it to search from the frontend".to_string()), + uid, actions: vec![Action::Search], - indexes: vec!["*".to_string()], + indexes: vec![StarOr::Star], expires_at: None, created_at: now, updated_at: now, @@ -128,19 +161,6 @@ impl Key { } } -/// Generate a printable key of 64 characters using thread_rng. -fn generate_id() -> [u8; KEY_ID_LENGTH] { - const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - let mut rng = rand::thread_rng(); - let mut bytes = [0; KEY_ID_LENGTH]; - for byte in bytes.iter_mut() { - *byte = CHARSET[rng.gen_range(0..CHARSET.len())]; - } - - bytes -} - fn parse_expiration_date(value: &Value) -> Result> { match value { Value::String(string) => OffsetDateTime::parse(string, &Rfc3339) diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index 22263735e..17f1a3567 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -5,18 +5,20 @@ mod key; mod store; use std::collections::{HashMap, HashSet}; +use std::ops::Deref; use std::path::Path; -use std::str::from_utf8; use std::sync::Arc; use serde::{Deserialize, Serialize}; use serde_json::Value; -use sha2::{Digest, Sha256}; use time::OffsetDateTime; +use uuid::Uuid; pub use action::{actions, Action}; use error::{AuthControllerError, Result}; pub use key::Key; +use meilisearch_types::star_or::StarOr; +use store::generate_key_as_hexa; pub use store::open_auth_store_env; use store::HeedAuthStore; @@ -42,62 +44,77 @@ impl AuthController { pub fn create_key(&self, value: Value) -> Result { let key = Key::create_from_value(value)?; - self.store.put_api_key(key) + match self.store.get_api_key(key.uid)? { + Some(_) => Err(AuthControllerError::ApiKeyAlreadyExists( + key.uid.to_string(), + )), + None => self.store.put_api_key(key), + } } - pub fn update_key(&self, key: impl AsRef, value: Value) -> Result { - let mut key = self.get_key(key)?; + pub fn update_key(&self, uid: Uuid, value: Value) -> Result { + let mut key = self.get_key(uid)?; key.update_from_value(value)?; self.store.put_api_key(key) } - pub fn get_key(&self, key: impl AsRef) -> Result { + pub fn get_key(&self, uid: Uuid) -> Result { self.store - .get_api_key(&key)? - .ok_or_else(|| AuthControllerError::ApiKeyNotFound(key.as_ref().to_string())) + .get_api_key(uid)? + .ok_or_else(|| AuthControllerError::ApiKeyNotFound(uid.to_string())) + } + + pub fn get_optional_uid_from_encoded_key(&self, encoded_key: &[u8]) -> Result> { + match &self.master_key { + Some(master_key) => self + .store + .get_uid_from_encoded_key(encoded_key, master_key.as_bytes()), + None => Ok(None), + } + } + + pub fn get_uid_from_encoded_key(&self, encoded_key: &str) -> Result { + self.get_optional_uid_from_encoded_key(encoded_key.as_bytes())? + .ok_or_else(|| AuthControllerError::ApiKeyNotFound(encoded_key.to_string())) } pub fn get_key_filters( &self, - key: impl AsRef, + uid: Uuid, search_rules: Option, ) -> Result { let mut filters = AuthFilter::default(); - if self - .master_key - .as_ref() - .map_or(false, |master_key| master_key != key.as_ref()) - { - let key = self - .store - .get_api_key(&key)? - .ok_or_else(|| AuthControllerError::ApiKeyNotFound(key.as_ref().to_string()))?; + let key = self + .store + .get_api_key(uid)? + .ok_or_else(|| AuthControllerError::ApiKeyNotFound(uid.to_string()))?; - if !key.indexes.iter().any(|i| i.as_str() == "*") { - filters.search_rules = match search_rules { - // Intersect search_rules with parent key authorized indexes. - Some(search_rules) => SearchRules::Map( - key.indexes - .into_iter() - .filter_map(|index| { - search_rules - .get_index_search_rules(&index) - .map(|index_search_rules| (index, Some(index_search_rules))) - }) - .collect(), - ), - None => SearchRules::Set(key.indexes.into_iter().collect()), - }; - } else if let Some(search_rules) = search_rules { - filters.search_rules = search_rules; - } - - filters.allow_index_creation = key - .actions - .iter() - .any(|&action| action == Action::IndexesAdd || action == Action::All); + if !key.indexes.iter().any(|i| i == &StarOr::Star) { + filters.search_rules = match search_rules { + // Intersect search_rules with parent key authorized indexes. + Some(search_rules) => SearchRules::Map( + key.indexes + .into_iter() + .filter_map(|index| { + search_rules.get_index_search_rules(index.deref()).map( + |index_search_rules| { + (String::from(index), Some(index_search_rules)) + }, + ) + }) + .collect(), + ), + None => SearchRules::Set(key.indexes.into_iter().map(String::from).collect()), + }; + } else if let Some(search_rules) = search_rules { + filters.search_rules = search_rules; } + filters.allow_index_creation = key + .actions + .iter() + .any(|&action| action == Action::IndexesAdd || action == Action::All); + Ok(filters) } @@ -105,13 +122,11 @@ impl AuthController { self.store.list_api_keys() } - pub fn delete_key(&self, key: impl AsRef) -> Result<()> { - if self.store.delete_api_key(&key)? { + pub fn delete_key(&self, uid: Uuid) -> Result<()> { + if self.store.delete_api_key(uid)? { Ok(()) } else { - Err(AuthControllerError::ApiKeyNotFound( - key.as_ref().to_string(), - )) + Err(AuthControllerError::ApiKeyNotFound(uid.to_string())) } } @@ -121,32 +136,32 @@ impl AuthController { /// Generate a valid key from a key id using the current master key. /// Returns None if no master key has been set. - pub fn generate_key(&self, id: &str) -> Option { + pub fn generate_key(&self, uid: Uuid) -> Option { self.master_key .as_ref() - .map(|master_key| generate_key(master_key.as_bytes(), id)) + .map(|master_key| generate_key_as_hexa(uid, master_key.as_bytes())) } /// Check if the provided key is authorized to make a specific action /// without checking if the key is valid. pub fn is_key_authorized( &self, - key: &[u8], + uid: Uuid, action: Action, index: Option<&str>, ) -> Result { match self .store // check if the key has access to all indexes. - .get_expiration_date(key, action, None)? + .get_expiration_date(uid, action, None)? .or(match index { // else check if the key has access to the requested index. Some(index) => { self.store - .get_expiration_date(key, action, Some(index.as_bytes()))? + .get_expiration_date(uid, action, Some(index.as_bytes()))? } // or to any index if no index has been requested. - None => self.store.prefix_first_expiration_date(key, action)?, + None => self.store.prefix_first_expiration_date(uid, action)?, }) { // check expiration date. Some(Some(exp)) => Ok(OffsetDateTime::now_utc() < exp), @@ -156,29 +171,6 @@ impl AuthController { None => Ok(false), } } - - /// Check if the provided key is valid - /// without checking if the key is authorized to make a specific action. - pub fn is_key_valid(&self, key: &[u8]) -> Result { - if let Some(id) = self.store.get_key_id(key) { - let id = from_utf8(&id)?; - if let Some(generated) = self.generate_key(id) { - return Ok(generated.as_bytes() == key); - } - } - - Ok(false) - } - - /// Check if the provided key is valid - /// and is authorized to make a specific action. - pub fn authenticate(&self, key: &[u8], action: Action, index: Option<&str>) -> Result { - if self.is_key_authorized(key, action, index)? { - self.is_key_valid(key) - } else { - Ok(false) - } - } } pub struct AuthFilter { @@ -258,12 +250,6 @@ pub struct IndexSearchRules { pub filter: Option, } -fn generate_key(master_key: &[u8], keyid: &str) -> String { - let key = [keyid.as_bytes(), master_key].concat(); - let sha = Sha256::digest(&key); - format!("{}{:x}", keyid, sha) -} - fn generate_default_keys(store: &HeedAuthStore) -> Result<()> { store.put_api_key(Key::default_admin())?; store.put_api_key(Key::default_search())?; diff --git a/meilisearch-auth/src/store.rs b/meilisearch-auth/src/store.rs index 4bd3cdded..49bbf356e 100644 --- a/meilisearch-auth/src/store.rs +++ b/meilisearch-auth/src/store.rs @@ -1,27 +1,32 @@ -use enum_iterator::IntoEnumIterator; use std::borrow::Cow; use std::cmp::Reverse; use std::convert::TryFrom; use std::convert::TryInto; use std::fs::create_dir_all; +use std::ops::Deref; use std::path::Path; use std::str; use std::sync::Arc; +use enum_iterator::IntoEnumIterator; +use hmac::{Hmac, Mac}; +use meilisearch_types::star_or::StarOr; use milli::heed::types::{ByteSlice, DecodeIgnore, SerdeJson}; use milli::heed::{Database, Env, EnvOpenOptions, RwTxn}; +use sha2::Sha256; use time::OffsetDateTime; +use uuid::fmt::Hyphenated; +use uuid::Uuid; use super::error::Result; use super::{Action, Key}; const AUTH_STORE_SIZE: usize = 1_073_741_824; //1GiB -pub const KEY_ID_LENGTH: usize = 8; const AUTH_DB_PATH: &str = "auth"; const KEY_DB_NAME: &str = "api-keys"; const KEY_ID_ACTION_INDEX_EXPIRATION_DB_NAME: &str = "keyid-action-index-expiration"; -pub type KeyId = [u8; KEY_ID_LENGTH]; +pub type KeyId = Uuid; #[derive(Clone)] pub struct HeedAuthStore { @@ -73,12 +78,13 @@ impl HeedAuthStore { } pub fn put_api_key(&self, key: Key) -> Result { + let uid = key.uid; let mut wtxn = self.env.write_txn()?; - self.keys.put(&mut wtxn, &key.id, &key)?; - let id = key.id; + self.keys.put(&mut wtxn, uid.as_bytes(), &key)?; + // delete key from inverted database before refilling it. - self.delete_key_from_inverted_db(&mut wtxn, &id)?; + self.delete_key_from_inverted_db(&mut wtxn, &uid)?; // create inverted database. let db = self.action_keyid_index_expiration; @@ -89,17 +95,17 @@ impl HeedAuthStore { key.actions.clone() }; - let no_index_restriction = key.indexes.contains(&"*".to_owned()); + let no_index_restriction = key.indexes.contains(&StarOr::Star); for action in actions { if no_index_restriction { // If there is no index restriction we put None. - db.put(&mut wtxn, &(&id, &action, None), &key.expires_at)?; + db.put(&mut wtxn, &(&uid, &action, None), &key.expires_at)?; } else { // else we create a key for each index. for index in key.indexes.iter() { db.put( &mut wtxn, - &(&id, &action, Some(index.as_bytes())), + &(&uid, &action, Some(index.deref().as_bytes())), &key.expires_at, )?; } @@ -111,24 +117,42 @@ impl HeedAuthStore { Ok(key) } - pub fn get_api_key(&self, key: impl AsRef) -> Result> { + pub fn get_api_key(&self, uid: Uuid) -> Result> { let rtxn = self.env.read_txn()?; - match self.get_key_id(key.as_ref().as_bytes()) { - Some(id) => self.keys.get(&rtxn, &id).map_err(|e| e.into()), - None => Ok(None), - } + self.keys.get(&rtxn, uid.as_bytes()).map_err(|e| e.into()) } - pub fn delete_api_key(&self, key: impl AsRef) -> Result { + pub fn get_uid_from_encoded_key( + &self, + encoded_key: &[u8], + master_key: &[u8], + ) -> Result> { + let rtxn = self.env.read_txn()?; + let uid = self + .keys + .remap_data_type::() + .iter(&rtxn)? + .filter_map(|res| match res { + Ok((uid, _)) => { + let (uid, _) = try_split_array_at(uid)?; + let uid = Uuid::from_bytes(*uid); + if generate_key_as_hexa(uid, master_key).as_bytes() == encoded_key { + Some(uid) + } else { + None + } + } + Err(_) => None, + }) + .next(); + + Ok(uid) + } + + pub fn delete_api_key(&self, uid: Uuid) -> Result { let mut wtxn = self.env.write_txn()?; - let existing = match self.get_key_id(key.as_ref().as_bytes()) { - Some(id) => { - let existing = self.keys.delete(&mut wtxn, &id)?; - self.delete_key_from_inverted_db(&mut wtxn, &id)?; - existing - } - None => false, - }; + let existing = self.keys.delete(&mut wtxn, uid.as_bytes())?; + self.delete_key_from_inverted_db(&mut wtxn, &uid)?; wtxn.commit()?; Ok(existing) @@ -147,49 +171,37 @@ impl HeedAuthStore { pub fn get_expiration_date( &self, - key: &[u8], + uid: Uuid, action: Action, index: Option<&[u8]>, ) -> Result>> { let rtxn = self.env.read_txn()?; - match self.get_key_id(key) { - Some(id) => { - let tuple = (&id, &action, index); - Ok(self.action_keyid_index_expiration.get(&rtxn, &tuple)?) - } - None => Ok(None), - } + let tuple = (&uid, &action, index); + Ok(self.action_keyid_index_expiration.get(&rtxn, &tuple)?) } pub fn prefix_first_expiration_date( &self, - key: &[u8], + uid: Uuid, action: Action, ) -> Result>> { let rtxn = self.env.read_txn()?; - match self.get_key_id(key) { - Some(id) => { - let tuple = (&id, &action, None); - Ok(self - .action_keyid_index_expiration - .prefix_iter(&rtxn, &tuple)? - .next() - .transpose()? - .map(|(_, expiration)| expiration)) - } - None => Ok(None), - } - } + let tuple = (&uid, &action, None); + let exp = self + .action_keyid_index_expiration + .prefix_iter(&rtxn, &tuple)? + .next() + .transpose()? + .map(|(_, expiration)| expiration); - pub fn get_key_id(&self, key: &[u8]) -> Option { - try_split_array_at::<_, KEY_ID_LENGTH>(key).map(|(id, _)| *id) + Ok(exp) } fn delete_key_from_inverted_db(&self, wtxn: &mut RwTxn, key: &KeyId) -> Result<()> { let mut iter = self .action_keyid_index_expiration .remap_types::() - .prefix_iter_mut(wtxn, key)?; + .prefix_iter_mut(wtxn, key.as_bytes())?; while iter.next().transpose()?.is_some() { // safety: we don't keep references from inside the LMDB database. unsafe { iter.del_current()? }; @@ -200,21 +212,22 @@ impl HeedAuthStore { } /// Codec allowing to retrieve the expiration date of an action, -/// optionnally on a spcific index, for a given key. +/// optionally on a specific index, for a given key. pub struct KeyIdActionCodec; impl<'a> milli::heed::BytesDecode<'a> for KeyIdActionCodec { type DItem = (KeyId, Action, Option<&'a [u8]>); fn bytes_decode(bytes: &'a [u8]) -> Option { - let (key_id, action_bytes) = try_split_array_at(bytes)?; + let (key_id_bytes, action_bytes) = try_split_array_at(bytes)?; let (action_bytes, index) = match try_split_array_at(action_bytes)? { (action, []) => (action, None), (action, index) => (action, Some(index)), }; + let key_id = Uuid::from_bytes(*key_id_bytes); let action = Action::from_repr(u8::from_be_bytes(*action_bytes))?; - Some((*key_id, action, index)) + Some((key_id, action, index)) } } @@ -224,7 +237,7 @@ impl<'a> milli::heed::BytesEncode<'a> for KeyIdActionCodec { fn bytes_encode((key_id, action, index): &Self::EItem) -> Option> { let mut bytes = Vec::new(); - bytes.extend_from_slice(*key_id); + bytes.extend_from_slice(key_id.as_bytes()); let action_bytes = u8::to_be_bytes(action.repr()); bytes.extend_from_slice(&action_bytes); if let Some(index) = index { @@ -235,6 +248,19 @@ impl<'a> milli::heed::BytesEncode<'a> for KeyIdActionCodec { } } +pub fn generate_key_as_hexa(uid: Uuid, master_key: &[u8]) -> String { + // format uid as hyphenated allowing user to generate their own keys. + let mut uid_buffer = [0; Hyphenated::LENGTH]; + let uid = uid.hyphenated().encode_lower(&mut uid_buffer); + + // new_from_slice function never fail. + let mut mac = Hmac::::new_from_slice(master_key).unwrap(); + mac.update(uid.as_bytes()); + + let result = mac.finalize(); + format!("{:x}", result.into_bytes()) +} + /// Divides one slice into two at an index, returns `None` if mid is out of bounds. pub fn try_split_at(slice: &[T], mid: usize) -> Option<(&[T], &[T])> { if mid <= slice.len() { diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 4711df2be..6af4dce48 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -4,7 +4,7 @@ description = "Meilisearch HTTP server" edition = "2021" license = "MIT" name = "meilisearch-http" -version = "0.27.2" +version = "0.28.0" [[bin]] name = "meilisearch" @@ -45,7 +45,7 @@ itertools = "0.10.3" jsonwebtoken = "8.0.1" log = "0.4.14" meilisearch-auth = { path = "../meilisearch-auth" } -meilisearch-error = { path = "../meilisearch-error" } +meilisearch-types = { path = "../meilisearch-types" } meilisearch-lib = { path = "../meilisearch-lib" } mime = "0.3.16" num_cpus = "1.13.1" @@ -57,10 +57,12 @@ platform-dirs = "0.3.0" rand = "0.8.5" rayon = "1.5.1" regex = "1.5.5" +reqwest = { version = "0.11.4", features = ["rustls-tls", "json"], default-features = false } rustls = "0.20.4" rustls-pemfile = "0.3.0" segment = { version = "0.2.0", optional = true } serde = { version = "1.0.136", features = ["derive"] } +serde-cs = "0.2.3" serde_json = { version = "1.0.79", features = ["preserve_order"] } sha2 = "0.10.2" siphasher = "0.3.10" @@ -73,16 +75,16 @@ thiserror = "1.0.30" time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } tokio = { version = "1.17.0", features = ["full"] } tokio-stream = "0.1.8" -uuid = { version = "0.8.2", features = ["serde"] } +uuid = { version = "1.1.2", features = ["serde", "v4"] } walkdir = "2.3.2" [dev-dependencies] actix-rt = "2.7.0" assert-json-diff = "2.0.1" +manifest-dir-macros = "0.1.14" maplit = "1.0.2" -paste = "1.0.6" -serde_url_params = "0.2.1" urlencoding = "2.1.0" +yaup = "0.2.0" [features] default = ["analytics", "mini-dashboard"] @@ -103,5 +105,5 @@ mini-dashboard = [ tikv-jemallocator = "0.4.3" [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.10/build.zip" -sha1 = "1adf96592c267425c110bfefc36b7fc6bfb0f93d" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.0/build.zip" +sha1 = "25d1615c608541375a08bd722c3fd3315f926be6" diff --git a/meilisearch-http/src/analytics/mod.rs b/meilisearch-http/src/analytics/mod.rs index 1d37a053d..b51f306a9 100644 --- a/meilisearch-http/src/analytics/mod.rs +++ b/meilisearch-http/src/analytics/mod.rs @@ -61,7 +61,7 @@ pub trait Analytics: Sync + Send { /// The method used to publish most analytics that do not need to be batched every hours fn publish(&self, event_name: String, send: Value, request: Option<&HttpRequest>); - /// This method should be called to aggergate a get search + /// This method should be called to aggregate a get search fn get_search(&self, aggregate: SearchAggregator); /// This method should be called to aggregate a post search diff --git a/meilisearch-http/src/analytics/segment_analytics.rs b/meilisearch-http/src/analytics/segment_analytics.rs index 3d3b23d70..b04d814aa 100644 --- a/meilisearch-http/src/analytics/segment_analytics.rs +++ b/meilisearch-http/src/analytics/segment_analytics.rs @@ -31,6 +31,8 @@ use crate::Opt; use super::{config_user_id_path, MEILISEARCH_CONFIG_PATH}; +const ANALYTICS_HEADER: &str = "X-Meilisearch-Client"; + /// Write the instance-uid in the `data.ms` and in `~/.config/MeiliSearch/path-to-db-instance-uid`. Ignore the errors. fn write_user_id(db_path: &Path, user_id: &str) { let _ = fs::write(db_path.join("instance-uid"), user_id.as_bytes()); @@ -48,7 +50,8 @@ const SEGMENT_API_KEY: &str = "P3FWhhEsJiEDCuEHpmcN9DHcK4hVfBvb"; pub fn extract_user_agents(request: &HttpRequest) -> Vec { request .headers() - .get(USER_AGENT) + .get(ANALYTICS_HEADER) + .or_else(|| request.headers().get(USER_AGENT)) .map(|header| header.to_str().ok()) .flatten() .unwrap_or("unknown") @@ -78,7 +81,19 @@ impl SegmentAnalytics { let user_id = user_id.unwrap_or_else(|| Uuid::new_v4().to_string()); write_user_id(&opt.db_path, &user_id); - let client = HttpClient::default(); + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .build(); + + // if reqwest throws an error we won't be able to send analytics + if client.is_err() { + return super::MockAnalytics::new(opt); + } + + let client = HttpClient::new( + client.unwrap(), + "https://telemetry.meilisearch.com".to_string(), + ); let user = User::UserId { user_id }; let mut batcher = AutoBatcher::new(client, Batcher::new(None), SEGMENT_API_KEY.to_string()); @@ -130,11 +145,7 @@ impl SegmentAnalytics { impl super::Analytics for SegmentAnalytics { fn publish(&self, event_name: String, mut send: Value, request: Option<&HttpRequest>) { - let user_agent = request - .map(|req| req.headers().get(USER_AGENT)) - .flatten() - .map(|header| header.to_str().unwrap_or("unknown")) - .map(|s| s.split(';').map(str::trim).collect::>()); + let user_agent = request.map(|req| extract_user_agents(req)); send["user-agent"] = json!(user_agent); let event = Track { @@ -363,7 +374,7 @@ pub struct SearchAggregator { highlight_pre_tag: bool, highlight_post_tag: bool, crop_marker: bool, - matches: bool, + show_matches_position: bool, crop_length: bool, } @@ -415,11 +426,11 @@ impl SearchAggregator { ret.max_limit = query.limit; ret.max_offset = query.offset.unwrap_or_default(); - ret.highlight_pre_tag = query.highlight_pre_tag != DEFAULT_HIGHLIGHT_PRE_TAG; - ret.highlight_post_tag = query.highlight_post_tag != DEFAULT_HIGHLIGHT_POST_TAG; - ret.crop_marker = query.crop_marker != DEFAULT_CROP_MARKER; - ret.crop_length = query.crop_length != DEFAULT_CROP_LENGTH; - ret.matches = query.matches; + ret.highlight_pre_tag = query.highlight_pre_tag != DEFAULT_HIGHLIGHT_PRE_TAG(); + ret.highlight_post_tag = query.highlight_post_tag != DEFAULT_HIGHLIGHT_POST_TAG(); + ret.crop_marker = query.crop_marker != DEFAULT_CROP_MARKER(); + ret.crop_length = query.crop_length != DEFAULT_CROP_LENGTH(); + ret.show_matches_position = query.show_matches_position; ret } @@ -472,7 +483,7 @@ impl SearchAggregator { self.highlight_pre_tag |= other.highlight_pre_tag; self.highlight_post_tag |= other.highlight_post_tag; self.crop_marker |= other.crop_marker; - self.matches |= other.matches; + self.show_matches_position |= other.show_matches_position; self.crop_length |= other.crop_length; } @@ -484,7 +495,7 @@ impl SearchAggregator { let percentile_99th = 0.99 * (self.total_succeeded as f64 - 1.) + 1.; // we get all the values in a sorted manner let time_spent = self.time_spent.into_sorted_vec(); - // We are only intersted by the slowest value of the 99th fastest results + // We are only interested by the slowest value of the 99th fastest results let time_spent = time_spent.get(percentile_99th as usize); let properties = json!({ @@ -515,7 +526,7 @@ impl SearchAggregator { "highlight_pre_tag": self.highlight_pre_tag, "highlight_post_tag": self.highlight_post_tag, "crop_marker": self.crop_marker, - "matches": self.matches, + "show_matches_position": self.show_matches_position, "crop_length": self.crop_length, }, }); @@ -563,8 +574,8 @@ impl DocumentsAggregator { let content_type = request .headers() .get(CONTENT_TYPE) - .map(|s| s.to_str().unwrap_or("unkown")) - .unwrap_or("unkown") + .and_then(|s| s.to_str().ok()) + .unwrap_or("unknown") .to_string(); ret.content_types.insert(content_type); ret.index_creation = index_creation; @@ -580,13 +591,13 @@ impl DocumentsAggregator { self.updated |= other.updated; // we can't create a union because there is no `into_union` method - for user_agent in other.user_agents.into_iter() { + for user_agent in other.user_agents { self.user_agents.insert(user_agent); } - for primary_key in other.primary_keys.into_iter() { + for primary_key in other.primary_keys { self.primary_keys.insert(primary_key); } - for content_type in other.content_types.into_iter() { + for content_type in other.content_types { self.content_types.insert(content_type); } self.index_creation |= other.index_creation; diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index b2b6c1b3c..86b7c1964 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -1,6 +1,6 @@ use actix_web as aweb; use aweb::error::{JsonPayloadError, QueryPayloadError}; -use meilisearch_error::{Code, ErrorCode, ResponseError}; +use meilisearch_types::error::{Code, ErrorCode, ResponseError}; #[derive(Debug, thiserror::Error)] pub enum MeilisearchHttpError { diff --git a/meilisearch-http/src/extractors/authentication/error.rs b/meilisearch-http/src/extractors/authentication/error.rs index 6d362dcbf..bb78c53d0 100644 --- a/meilisearch-http/src/extractors/authentication/error.rs +++ b/meilisearch-http/src/extractors/authentication/error.rs @@ -1,4 +1,4 @@ -use meilisearch_error::{Code, ErrorCode}; +use meilisearch_types::error::{Code, ErrorCode}; #[derive(Debug, thiserror::Error)] pub enum AuthenticationError { diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index c4cd9ef14..22f080a6f 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -5,12 +5,11 @@ use std::ops::Deref; use std::pin::Pin; use actix_web::FromRequest; +use error::AuthenticationError; use futures::future::err; use futures::Future; -use meilisearch_error::{Code, ResponseError}; - -use error::AuthenticationError; use meilisearch_auth::{AuthController, AuthFilter}; +use meilisearch_types::error::{Code, ResponseError}; pub struct GuardedData { data: D, @@ -132,6 +131,7 @@ pub mod policies { use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation}; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; + use uuid::Uuid; use crate::extractors::authentication::Policy; use meilisearch_auth::{Action, AuthController, AuthFilter, SearchRules}; @@ -146,34 +146,21 @@ pub mod policies { validation } - /// Extracts the key prefix used to sign the payload from the payload, without performing any validation. - fn extract_key_prefix(token: &str) -> Option { + /// Extracts the key id used to sign the payload, without performing any validation. + fn extract_key_id(token: &str) -> Option { let mut validation = tenant_token_validation(); validation.insecure_disable_signature_validation(); let dummy_key = DecodingKey::from_secret(b"secret"); let token_data = decode::(token, &dummy_key, &validation).ok()?; // get token fields without validating it. - let Claims { api_key_prefix, .. } = token_data.claims; - Some(api_key_prefix) + let Claims { api_key_uid, .. } = token_data.claims; + Some(api_key_uid) } - pub struct MasterPolicy; - - impl Policy for MasterPolicy { - fn authenticate( - auth: AuthController, - token: &str, - _index: Option<&str>, - ) -> Option { - if let Some(master_key) = auth.get_master_key() { - if master_key == token { - return Some(AuthFilter::default()); - } - } - - None - } + fn is_keys_action(action: u8) -> bool { + use actions::*; + matches!(action, KEYS_GET | KEYS_CREATE | KEYS_UPDATE | KEYS_DELETE) } pub struct ActionPolicy; @@ -185,7 +172,12 @@ pub mod policies { index: Option<&str>, ) -> Option { // authenticate if token is the master key. - if auth.get_master_key().map_or(true, |mk| mk == token) { + // master key can only have access to keys routes. + // if master key is None only keys routes are inaccessible. + if auth + .get_master_key() + .map_or_else(|| !is_keys_action(A), |mk| mk == token) + { return Some(AuthFilter::default()); } @@ -195,8 +187,10 @@ pub mod policies { return Some(filters); } else if let Some(action) = Action::from_repr(A) { // API key - if let Ok(true) = auth.authenticate(token.as_bytes(), action, index) { - return auth.get_key_filters(token, None).ok(); + if let Ok(Some(uid)) = auth.get_optional_uid_from_encoded_key(token.as_bytes()) { + if let Ok(true) = auth.is_key_authorized(uid, action, index) { + return auth.get_key_filters(uid, None).ok(); + } } } @@ -215,14 +209,11 @@ pub mod policies { return None; } - let api_key_prefix = extract_key_prefix(token)?; + let uid = extract_key_id(token)?; // check if parent key is authorized to do the action. - if auth - .is_key_authorized(api_key_prefix.as_bytes(), Action::Search, index) - .ok()? - { + if auth.is_key_authorized(uid, Action::Search, index).ok()? { // Check if tenant token is valid. - let key = auth.generate_key(&api_key_prefix)?; + let key = auth.generate_key(uid)?; let data = decode::( token, &DecodingKey::from_secret(key.as_bytes()), @@ -245,7 +236,7 @@ pub mod policies { } return auth - .get_key_filters(api_key_prefix, Some(data.claims.search_rules)) + .get_key_filters(uid, Some(data.claims.search_rules)) .ok(); } @@ -258,6 +249,6 @@ pub mod policies { struct Claims { search_rules: SearchRules, exp: Option, - api_key_prefix: String, + api_key_uid: Uuid, } } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index d1f5d9da1..6485784fc 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -2,7 +2,7 @@ #[macro_use] pub mod error; pub mod analytics; -mod task; +pub mod task; #[macro_use] pub mod extractors; pub mod helpers; @@ -31,7 +31,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result { let mut meilisearch = MeiliSearch::builder(); // enable autobatching? - let _ = AUTOBATCHING_ENABLED.store( + AUTOBATCHING_ENABLED.store( opt.scheduler_options.enable_auto_batching, std::sync::atomic::Ordering::Relaxed, ); @@ -148,10 +148,10 @@ macro_rules! create_app { use actix_web::middleware::TrailingSlash; use actix_web::App; use actix_web::{middleware, web}; - use meilisearch_error::ResponseError; use meilisearch_http::error::MeilisearchHttpError; use meilisearch_http::routes; use meilisearch_http::{configure_data, dashboard}; + use meilisearch_types::error::ResponseError; App::new() .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics)) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 498bbb82d..f903663eb 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -1,6 +1,7 @@ use std::env; use std::sync::Arc; +use actix_web::http::KeepAlive; use actix_web::HttpServer; use clap::Parser; use meilisearch_auth::AuthController; @@ -83,7 +84,8 @@ async fn run_http( ) }) // Disable signals allows the server to terminate immediately when a user enter CTRL-C - .disable_signals(); + .disable_signals() + .keep_alive(KeepAlive::Os); if let Some(config) = opt.get_ssl_config()? { http_server diff --git a/meilisearch-http/src/routes/api_key.rs b/meilisearch-http/src/routes/api_key.rs index 310b09c4d..7605fa644 100644 --- a/meilisearch-http/src/routes/api_key.rs +++ b/meilisearch-http/src/routes/api_key.rs @@ -1,17 +1,19 @@ use std::str; use actix_web::{web, HttpRequest, HttpResponse}; - -use meilisearch_auth::{error::AuthControllerError, Action, AuthController, Key}; use serde::{Deserialize, Serialize}; use serde_json::Value; use time::OffsetDateTime; +use uuid::Uuid; + +use meilisearch_auth::{error::AuthControllerError, Action, AuthController, Key}; +use meilisearch_types::error::{Code, ResponseError}; use crate::extractors::{ authentication::{policies::*, GuardedData}, sequential_extractor::SeqHandler, }; -use meilisearch_error::{Code, ResponseError}; +use crate::routes::Pagination; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( @@ -20,7 +22,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .route(web::get().to(SeqHandler(list_api_keys))), ) .service( - web::resource("/{api_key}") + web::resource("/{key}") .route(web::get().to(SeqHandler(get_api_key))) .route(web::patch().to(SeqHandler(patch_api_key))) .route(web::delete().to(SeqHandler(delete_api_key))), @@ -28,7 +30,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { } pub async fn create_api_key( - auth_controller: GuardedData, + auth_controller: GuardedData, AuthController>, body: web::Json, _req: HttpRequest, ) -> Result { @@ -44,30 +46,35 @@ pub async fn create_api_key( } pub async fn list_api_keys( - auth_controller: GuardedData, - _req: HttpRequest, + auth_controller: GuardedData, AuthController>, + paginate: web::Query, ) -> Result { - let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { + let page_view = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { let keys = auth_controller.list_keys()?; - let res: Vec<_> = keys - .into_iter() - .map(|k| KeyView::from_key(k, &auth_controller)) - .collect(); - Ok(res) + let page_view = paginate.auto_paginate_sized( + keys.into_iter() + .map(|k| KeyView::from_key(k, &auth_controller)), + ); + + Ok(page_view) }) .await .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; - Ok(HttpResponse::Ok().json(KeyListView::from(res))) + Ok(HttpResponse::Ok().json(page_view)) } pub async fn get_api_key( - auth_controller: GuardedData, + auth_controller: GuardedData, AuthController>, path: web::Path, ) -> Result { - let api_key = path.into_inner().api_key; + let key = path.into_inner().key; + let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { - let key = auth_controller.get_key(&api_key)?; + let uid = + Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?; + let key = auth_controller.get_key(uid)?; + Ok(KeyView::from_key(key, &auth_controller)) }) .await @@ -77,14 +84,17 @@ pub async fn get_api_key( } pub async fn patch_api_key( - auth_controller: GuardedData, + auth_controller: GuardedData, AuthController>, body: web::Json, path: web::Path, ) -> Result { - let api_key = path.into_inner().api_key; + let key = path.into_inner().key; let body = body.into_inner(); let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { - let key = auth_controller.update_key(&api_key, body)?; + let uid = + Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?; + let key = auth_controller.update_key(uid, body)?; + Ok(KeyView::from_key(key, &auth_controller)) }) .await @@ -94,27 +104,33 @@ pub async fn patch_api_key( } pub async fn delete_api_key( - auth_controller: GuardedData, + auth_controller: GuardedData, AuthController>, path: web::Path, ) -> Result { - let api_key = path.into_inner().api_key; - tokio::task::spawn_blocking(move || auth_controller.delete_key(&api_key)) - .await - .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; + let key = path.into_inner().key; + tokio::task::spawn_blocking(move || { + let uid = + Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?; + auth_controller.delete_key(uid) + }) + .await + .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; Ok(HttpResponse::NoContent().finish()) } #[derive(Deserialize)] pub struct AuthParam { - api_key: String, + key: String, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct KeyView { + name: Option, description: Option, key: String, + uid: Uuid, actions: Vec, indexes: Vec, #[serde(serialize_with = "time::serde::rfc3339::option::serialize")] @@ -127,28 +143,18 @@ struct KeyView { impl KeyView { fn from_key(key: Key, auth: &AuthController) -> Self { - let key_id = str::from_utf8(&key.id).unwrap(); - let generated_key = auth.generate_key(key_id).unwrap_or_default(); + let generated_key = auth.generate_key(key.uid).unwrap_or_default(); KeyView { + name: key.name, description: key.description, key: generated_key, + uid: key.uid, actions: key.actions, - indexes: key.indexes, + indexes: key.indexes.into_iter().map(String::from).collect(), expires_at: key.expires_at, created_at: key.created_at, updated_at: key.updated_at, } } } - -#[derive(Debug, Serialize)] -struct KeyListView { - results: Vec, -} - -impl From> for KeyListView { - fn from(results: Vec) -> Self { - Self { results } - } -} diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index 65cd7521f..4d9106ee0 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -1,19 +1,16 @@ use actix_web::{web, HttpRequest, HttpResponse}; use log::debug; -use meilisearch_error::ResponseError; use meilisearch_lib::MeiliSearch; -use serde::{Deserialize, Serialize}; +use meilisearch_types::error::ResponseError; use serde_json::json; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::sequential_extractor::SeqHandler; +use crate::task::SummarizedTaskView; pub fn configure(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump)))) - .service( - web::resource("/{dump_uid}/status").route(web::get().to(SeqHandler(get_dump_status))), - ); + cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump)))); } pub async fn create_dump( @@ -23,29 +20,8 @@ pub async fn create_dump( ) -> Result { analytics.publish("Dump Created".to_string(), json!({}), Some(&req)); - let res = meilisearch.create_dump().await?; + let res: SummarizedTaskView = meilisearch.register_dump_task().await?.into(); debug!("returns: {:?}", res); Ok(HttpResponse::Accepted().json(res)) } - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DumpStatusResponse { - status: String, -} - -#[derive(Deserialize)] -struct DumpParam { - dump_uid: String, -} - -async fn get_dump_status( - meilisearch: GuardedData, MeiliSearch>, - path: web::Path, -) -> Result { - let res = meilisearch.dump_info(path.dump_uid.clone()).await?; - - debug!("returns: {:?}", res); - Ok(HttpResponse::Ok().json(res)) -} diff --git a/meilisearch-http/src/routes/indexes/documents.rs b/meilisearch-http/src/routes/indexes/documents.rs index 66551ec77..2becc6db1 100644 --- a/meilisearch-http/src/routes/indexes/documents.rs +++ b/meilisearch-http/src/routes/indexes/documents.rs @@ -6,13 +6,15 @@ use actix_web::{web, HttpRequest, HttpResponse}; use bstr::ByteSlice; use futures::{Stream, StreamExt}; use log::debug; -use meilisearch_error::ResponseError; use meilisearch_lib::index_controller::{DocumentAdditionFormat, Update}; use meilisearch_lib::milli::update::IndexDocumentsMethod; use meilisearch_lib::MeiliSearch; +use meilisearch_types::error::ResponseError; +use meilisearch_types::star_or::StarOr; use mime::Mime; use once_cell::sync::Lazy; use serde::Deserialize; +use serde_cs::vec::CS; use serde_json::Value; use tokio::sync::mpsc; @@ -21,11 +23,9 @@ use crate::error::MeilisearchHttpError; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::payload::Payload; use crate::extractors::sequential_extractor::SeqHandler; +use crate::routes::{fold_star_or, PaginationView}; use crate::task::SummarizedTaskView; -const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; -const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20; - static ACCEPTED_CONTENT_TYPE: Lazy> = Lazy::new(|| { vec![ "application/json".to_string(), @@ -86,14 +86,24 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GetDocument { + fields: Option>>, +} + pub async fn get_document( meilisearch: GuardedData, MeiliSearch>, path: web::Path, + params: web::Query, ) -> Result { let index = path.index_uid.clone(); let id = path.document_id.clone(); + let GetDocument { fields } = params.into_inner(); + let attributes_to_retrieve = fields.and_then(fold_star_or); + let document = meilisearch - .document(index, id, None as Option>) + .document(index, id, attributes_to_retrieve) .await?; debug!("returns: {:?}", document); Ok(HttpResponse::Ok().json(document)) @@ -116,9 +126,11 @@ pub async fn delete_document( #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct BrowseQuery { - offset: Option, - limit: Option, - attributes_to_retrieve: Option, + #[serde(default)] + offset: usize, + #[serde(default = "crate::routes::PAGINATION_DEFAULT_LIMIT")] + limit: usize, + fields: Option>>, } pub async fn get_all_documents( @@ -127,27 +139,21 @@ pub async fn get_all_documents( params: web::Query, ) -> Result { debug!("called with params: {:?}", params); - let attributes_to_retrieve = params.attributes_to_retrieve.as_ref().and_then(|attrs| { - let mut names = Vec::new(); - for name in attrs.split(',').map(String::from) { - if name == "*" { - return None; - } - names.push(name); - } - Some(names) - }); + let BrowseQuery { + limit, + offset, + fields, + } = params.into_inner(); + let attributes_to_retrieve = fields.and_then(fold_star_or); - let documents = meilisearch - .documents( - path.into_inner(), - params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET), - params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT), - attributes_to_retrieve, - ) + let (total, documents) = meilisearch + .documents(path.into_inner(), offset, limit, attributes_to_retrieve) .await?; - debug!("returns: {:?}", documents); - Ok(HttpResponse::Ok().json(documents)) + + let ret = PaginationView::new(offset, limit, total as usize, documents); + + debug!("returns: {:?}", ret); + Ok(HttpResponse::Ok().json(ret)) } #[derive(Deserialize, Debug)] diff --git a/meilisearch-http/src/routes/indexes/mod.rs b/meilisearch-http/src/routes/indexes/mod.rs index bd74fd724..ed6196ebd 100644 --- a/meilisearch-http/src/routes/indexes/mod.rs +++ b/meilisearch-http/src/routes/indexes/mod.rs @@ -1,8 +1,8 @@ use actix_web::{web, HttpRequest, HttpResponse}; use log::debug; -use meilisearch_error::ResponseError; use meilisearch_lib::index_controller::Update; use meilisearch_lib::MeiliSearch; +use meilisearch_types::error::ResponseError; use serde::{Deserialize, Serialize}; use serde_json::json; use time::OffsetDateTime; @@ -12,10 +12,11 @@ use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::sequential_extractor::SeqHandler; use crate::task::SummarizedTaskView; +use super::Pagination; + pub mod documents; pub mod search; pub mod settings; -pub mod tasks; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( @@ -28,30 +29,32 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service( web::resource("") .route(web::get().to(SeqHandler(get_index))) - .route(web::put().to(SeqHandler(update_index))) + .route(web::patch().to(SeqHandler(update_index))) .route(web::delete().to(SeqHandler(delete_index))), ) .service(web::resource("/stats").route(web::get().to(SeqHandler(get_index_stats)))) .service(web::scope("/documents").configure(documents::configure)) .service(web::scope("/search").configure(search::configure)) - .service(web::scope("/tasks").configure(tasks::configure)) .service(web::scope("/settings").configure(settings::configure)), ); } pub async fn list_indexes( data: GuardedData, MeiliSearch>, + paginate: web::Query, ) -> Result { let search_rules = &data.filters().search_rules; - let indexes: Vec<_> = data - .list_indexes() - .await? + let indexes: Vec<_> = data.list_indexes().await?; + let nb_indexes = indexes.len(); + let iter = indexes .into_iter() - .filter(|i| search_rules.is_index_authorized(&i.uid)) - .collect(); + .filter(|i| search_rules.is_index_authorized(&i.uid)); + let ret = paginate + .into_inner() + .auto_paginate_unsized(nb_indexes, iter); - debug!("returns: {:?}", indexes); - Ok(HttpResponse::Ok().json(indexes)) + debug!("returns: {:?}", ret); + Ok(HttpResponse::Ok().json(ret)) } #[derive(Debug, Deserialize)] diff --git a/meilisearch-http/src/routes/indexes/search.rs b/meilisearch-http/src/routes/indexes/search.rs index 14d36c1b3..62bd65e14 100644 --- a/meilisearch-http/src/routes/indexes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -1,13 +1,14 @@ use actix_web::{web, HttpRequest, HttpResponse}; use log::debug; use meilisearch_auth::IndexSearchRules; -use meilisearch_error::ResponseError; use meilisearch_lib::index::{ - default_crop_length, default_crop_marker, default_highlight_post_tag, - default_highlight_pre_tag, SearchQuery, DEFAULT_SEARCH_LIMIT, + SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, + DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, }; use meilisearch_lib::MeiliSearch; +use meilisearch_types::error::ResponseError; use serde::Deserialize; +use serde_cs::vec::CS; use serde_json::Value; use crate::analytics::{Analytics, SearchAggregator}; @@ -28,42 +29,26 @@ pub struct SearchQueryGet { q: Option, offset: Option, limit: Option, - attributes_to_retrieve: Option, - attributes_to_crop: Option, - #[serde(default = "default_crop_length")] + attributes_to_retrieve: Option>, + attributes_to_crop: Option>, + #[serde(default = "DEFAULT_CROP_LENGTH")] crop_length: usize, - attributes_to_highlight: Option, + attributes_to_highlight: Option>, filter: Option, sort: Option, #[serde(default = "Default::default")] - matches: bool, - facets_distribution: Option, - #[serde(default = "default_highlight_pre_tag")] + show_matches_position: bool, + facets: Option>, + #[serde(default = "DEFAULT_HIGHLIGHT_PRE_TAG")] highlight_pre_tag: String, - #[serde(default = "default_highlight_post_tag")] + #[serde(default = "DEFAULT_HIGHLIGHT_POST_TAG")] highlight_post_tag: String, - #[serde(default = "default_crop_marker")] + #[serde(default = "DEFAULT_CROP_MARKER")] crop_marker: String, } impl From for SearchQuery { fn from(other: SearchQueryGet) -> Self { - let attributes_to_retrieve = other - .attributes_to_retrieve - .map(|attrs| attrs.split(',').map(String::from).collect()); - - let attributes_to_crop = other - .attributes_to_crop - .map(|attrs| attrs.split(',').map(String::from).collect()); - - let attributes_to_highlight = other - .attributes_to_highlight - .map(|attrs| attrs.split(',').map(String::from).collect()); - - let facets_distribution = other - .facets_distribution - .map(|attrs| attrs.split(',').map(String::from).collect()); - let filter = match other.filter { Some(f) => match serde_json::from_str(&f) { Ok(v) => Some(v), @@ -72,20 +57,22 @@ impl From for SearchQuery { None => None, }; - let sort = other.sort.map(|attr| fix_sort_query_parameters(&attr)); - Self { q: other.q, offset: other.offset, - limit: other.limit.unwrap_or(DEFAULT_SEARCH_LIMIT), - attributes_to_retrieve, - attributes_to_crop, + limit: other.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT), + attributes_to_retrieve: other + .attributes_to_retrieve + .map(|o| o.into_iter().collect()), + attributes_to_crop: other.attributes_to_crop.map(|o| o.into_iter().collect()), crop_length: other.crop_length, - attributes_to_highlight, + attributes_to_highlight: other + .attributes_to_highlight + .map(|o| o.into_iter().collect()), filter, - sort, - matches: other.matches, - facets_distribution, + sort: other.sort.map(|attr| fix_sort_query_parameters(&attr)), + show_matches_position: other.show_matches_position, + facets: other.facets.map(|o| o.into_iter().collect()), highlight_pre_tag: other.highlight_pre_tag, highlight_post_tag: other.highlight_post_tag, crop_marker: other.crop_marker, @@ -124,10 +111,9 @@ fn fix_sort_query_parameters(sort_query: &str) -> Vec { sort_parameters.push(current_sort.to_string()); merge = true; } else if merge && !sort_parameters.is_empty() { - sort_parameters - .last_mut() - .unwrap() - .push_str(&format!(",{}", current_sort)); + let s = sort_parameters.last_mut().unwrap(); + s.push(','); + s.push_str(current_sort); if current_sort.ends_with("):desc") || current_sort.ends_with("):asc") { merge = false; } @@ -169,10 +155,6 @@ pub async fn search_with_url_query( let search_result = search_result?; - // Tests that the nb_hits is always set to false - #[cfg(test)] - assert!(!search_result.exhaustive_nb_hits); - debug!("returns: {:?}", search_result); Ok(HttpResponse::Ok().json(search_result)) } @@ -207,10 +189,6 @@ pub async fn search_with_post( let search_result = search_result?; - // Tests that the nb_hits is always set to false - #[cfg(test)] - assert!(!search_result.exhaustive_nb_hits); - debug!("returns: {:?}", search_result); Ok(HttpResponse::Ok().json(search_result)) } diff --git a/meilisearch-http/src/routes/indexes/settings.rs b/meilisearch-http/src/routes/indexes/settings.rs index 222aca580..bc8642def 100644 --- a/meilisearch-http/src/routes/indexes/settings.rs +++ b/meilisearch-http/src/routes/indexes/settings.rs @@ -1,10 +1,10 @@ use log::debug; use actix_web::{web, HttpRequest, HttpResponse}; -use meilisearch_error::ResponseError; use meilisearch_lib::index::{Settings, Unchecked}; use meilisearch_lib::index_controller::Update; use meilisearch_lib::MeiliSearch; +use meilisearch_types::error::ResponseError; use serde_json::json; use crate::analytics::Analytics; @@ -13,7 +13,7 @@ use crate::task::SummarizedTaskView; #[macro_export] macro_rules! make_setting_route { - ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal, $analytics_var:ident, $analytics:expr) => { + ($route:literal, $update_verb:ident, $type:ty, $attr:ident, $camelcase_attr:literal, $analytics_var:ident, $analytics:expr) => { pub mod $attr { use actix_web::{web, HttpRequest, HttpResponse, Resource}; use log::debug; @@ -21,7 +21,7 @@ macro_rules! make_setting_route { use meilisearch_lib::milli::update::Setting; use meilisearch_lib::{index::Settings, index_controller::Update, MeiliSearch}; - use meilisearch_error::ResponseError; + use meilisearch_types::error::ResponseError; use $crate::analytics::Analytics; use $crate::extractors::authentication::{policies::*, GuardedData}; use $crate::extractors::sequential_extractor::SeqHandler; @@ -100,18 +100,27 @@ macro_rules! make_setting_route { pub fn resources() -> Resource { Resource::new($route) .route(web::get().to(SeqHandler(get))) - .route(web::post().to(SeqHandler(update))) + .route(web::$update_verb().to(SeqHandler(update))) .route(web::delete().to(SeqHandler(delete))) } } }; - ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => { - make_setting_route!($route, $type, $attr, $camelcase_attr, _analytics, |_, _| {}); + ($route:literal, $update_verb:ident, $type:ty, $attr:ident, $camelcase_attr:literal) => { + make_setting_route!( + $route, + $update_verb, + $type, + $attr, + $camelcase_attr, + _analytics, + |_, _| {} + ); }; } make_setting_route!( "/filterable-attributes", + put, std::collections::BTreeSet, filterable_attributes, "filterableAttributes", @@ -134,6 +143,7 @@ make_setting_route!( make_setting_route!( "/sortable-attributes", + put, std::collections::BTreeSet, sortable_attributes, "sortableAttributes", @@ -156,6 +166,7 @@ make_setting_route!( make_setting_route!( "/displayed-attributes", + put, Vec, displayed_attributes, "displayedAttributes" @@ -163,6 +174,7 @@ make_setting_route!( make_setting_route!( "/typo-tolerance", + patch, meilisearch_lib::index::updates::TypoSettings, typo_tolerance, "typoTolerance", @@ -204,6 +216,7 @@ make_setting_route!( make_setting_route!( "/searchable-attributes", + put, Vec, searchable_attributes, "searchableAttributes", @@ -225,6 +238,7 @@ make_setting_route!( make_setting_route!( "/stop-words", + put, std::collections::BTreeSet, stop_words, "stopWords" @@ -232,6 +246,7 @@ make_setting_route!( make_setting_route!( "/synonyms", + put, std::collections::BTreeMap>, synonyms, "synonyms" @@ -239,6 +254,7 @@ make_setting_route!( make_setting_route!( "/distinct-attribute", + put, String, distinct_attribute, "distinctAttribute" @@ -246,6 +262,7 @@ make_setting_route!( make_setting_route!( "/ranking-rules", + put, Vec, ranking_rules, "rankingRules", @@ -265,13 +282,57 @@ make_setting_route!( } ); +make_setting_route!( + "/faceting", + patch, + meilisearch_lib::index::updates::FacetingSettings, + faceting, + "faceting", + analytics, + |setting: &Option, req: &HttpRequest| { + use serde_json::json; + + analytics.publish( + "Faceting Updated".to_string(), + json!({ + "faceting": { + "max_values_per_facet": setting.as_ref().and_then(|s| s.max_values_per_facet.set()), + }, + }), + Some(req), + ); + } +); + +make_setting_route!( + "/pagination", + patch, + meilisearch_lib::index::updates::PaginationSettings, + pagination, + "pagination", + analytics, + |setting: &Option, req: &HttpRequest| { + use serde_json::json; + + analytics.publish( + "Pagination Updated".to_string(), + json!({ + "pagination": { + "max_total_hits": setting.as_ref().and_then(|s| s.max_total_hits.set()), + }, + }), + Some(req), + ); + } +); + macro_rules! generate_configure { ($($mod:ident),*) => { pub fn configure(cfg: &mut web::ServiceConfig) { use crate::extractors::sequential_extractor::SeqHandler; cfg.service( web::resource("") - .route(web::post().to(SeqHandler(update_all))) + .route(web::patch().to(SeqHandler(update_all))) .route(web::get().to(SeqHandler(get_all))) .route(web::delete().to(SeqHandler(delete_all)))) $(.service($mod::resources()))*; @@ -288,7 +349,9 @@ generate_configure!( stop_words, synonyms, ranking_rules, - typo_tolerance + typo_tolerance, + pagination, + faceting ); pub async fn update_all( @@ -348,6 +411,18 @@ pub async fn update_all( .map(|s| s.two_typos.set())) .flatten(), }, + "faceting": { + "max_values_per_facet": settings.faceting + .as_ref() + .set() + .and_then(|s| s.max_values_per_facet.as_ref().set()), + }, + "pagination": { + "max_total_hits": settings.pagination + .as_ref() + .set() + .and_then(|s| s.max_total_hits.as_ref().set()), + }, }), Some(&req), ); diff --git a/meilisearch-http/src/routes/indexes/tasks.rs b/meilisearch-http/src/routes/indexes/tasks.rs deleted file mode 100644 index 01ed85db8..000000000 --- a/meilisearch-http/src/routes/indexes/tasks.rs +++ /dev/null @@ -1,80 +0,0 @@ -use actix_web::{web, HttpRequest, HttpResponse}; -use log::debug; -use meilisearch_error::ResponseError; -use meilisearch_lib::MeiliSearch; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use time::OffsetDateTime; - -use crate::analytics::Analytics; -use crate::extractors::authentication::{policies::*, GuardedData}; -use crate::extractors::sequential_extractor::SeqHandler; -use crate::task::{TaskListView, TaskView}; - -pub fn configure(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("").route(web::get().to(SeqHandler(get_all_tasks_status)))) - .service(web::resource("{task_id}").route(web::get().to(SeqHandler(get_task_status)))); -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct UpdateIndexResponse { - name: String, - uid: String, - #[serde(serialize_with = "time::serde::rfc3339::serialize")] - created_at: OffsetDateTime, - #[serde(serialize_with = "time::serde::rfc3339::serialize")] - updated_at: OffsetDateTime, - #[serde(serialize_with = "time::serde::rfc3339::serialize")] - primary_key: OffsetDateTime, -} - -#[derive(Deserialize)] -pub struct UpdateParam { - index_uid: String, - task_id: u64, -} - -pub async fn get_task_status( - meilisearch: GuardedData, MeiliSearch>, - index_uid: web::Path, - req: HttpRequest, - analytics: web::Data, -) -> Result { - analytics.publish( - "Index Tasks Seen".to_string(), - json!({ "per_task_uid": true }), - Some(&req), - ); - - let UpdateParam { index_uid, task_id } = index_uid.into_inner(); - - let task: TaskView = meilisearch.get_index_task(index_uid, task_id).await?.into(); - - debug!("returns: {:?}", task); - Ok(HttpResponse::Ok().json(task)) -} - -pub async fn get_all_tasks_status( - meilisearch: GuardedData, MeiliSearch>, - index_uid: web::Path, - req: HttpRequest, - analytics: web::Data, -) -> Result { - analytics.publish( - "Index Tasks Seen".to_string(), - json!({ "per_task_uid": false }), - Some(&req), - ); - - let tasks: TaskListView = meilisearch - .list_index_task(index_uid.into_inner(), None, None) - .await? - .into_iter() - .map(TaskView::from) - .collect::>() - .into(); - - debug!("returns: {:?}", tasks); - Ok(HttpResponse::Ok().json(tasks)) -} diff --git a/meilisearch-http/src/routes/indexes/updates.rs b/meilisearch-http/src/routes/indexes/updates.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 49397444f..f61854c48 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,11 +1,13 @@ use actix_web::{web, HttpResponse}; use log::debug; use serde::{Deserialize, Serialize}; + use time::OffsetDateTime; -use meilisearch_error::ResponseError; use meilisearch_lib::index::{Settings, Unchecked}; use meilisearch_lib::MeiliSearch; +use meilisearch_types::error::ResponseError; +use meilisearch_types::star_or::StarOr; use crate::extractors::authentication::{policies::*, GuardedData}; @@ -24,6 +26,101 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::scope("/indexes").configure(indexes::configure)); } +/// Extracts the raw values from the `StarOr` types and +/// return None if a `StarOr::Star` is encountered. +pub fn fold_star_or(content: impl IntoIterator>) -> Option +where + O: FromIterator, +{ + content + .into_iter() + .map(|value| match value { + StarOr::Star => None, + StarOr::Other(val) => Some(val), + }) + .collect() +} + +const PAGINATION_DEFAULT_LIMIT: fn() -> usize = || 20; + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Pagination { + #[serde(default)] + pub offset: usize, + #[serde(default = "PAGINATION_DEFAULT_LIMIT")] + pub limit: usize, +} + +#[derive(Debug, Clone, Serialize)] +pub struct PaginationView { + pub results: Vec, + pub offset: usize, + pub limit: usize, + pub total: usize, +} + +impl Pagination { + /// Given the full data to paginate, returns the selected section. + pub fn auto_paginate_sized( + self, + content: impl IntoIterator + ExactSizeIterator, + ) -> PaginationView + where + T: Serialize, + { + let total = content.len(); + let content: Vec<_> = content + .into_iter() + .skip(self.offset) + .take(self.limit) + .collect(); + self.format_with(total, content) + } + + /// Given an iterator and the total number of elements, returns the selected section. + pub fn auto_paginate_unsized( + self, + total: usize, + content: impl IntoIterator, + ) -> PaginationView + where + T: Serialize, + { + let content: Vec<_> = content + .into_iter() + .skip(self.offset) + .take(self.limit) + .collect(); + self.format_with(total, content) + } + + /// Given the data already paginated + the total number of elements, it stores + /// everything in a [PaginationResult]. + pub fn format_with(self, total: usize, results: Vec) -> PaginationView + where + T: Serialize, + { + PaginationView { + results, + offset: self.offset, + limit: self.limit, + total, + } + } +} + +impl PaginationView { + pub fn new(offset: usize, limit: usize, total: usize, results: Vec) -> Self { + Self { + offset, + limit, + results, + total, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[allow(clippy::large_enum_variant)] #[serde(tag = "name")] diff --git a/meilisearch-http/src/routes/tasks.rs b/meilisearch-http/src/routes/tasks.rs index ae932253a..016aadfb8 100644 --- a/meilisearch-http/src/routes/tasks.rs +++ b/meilisearch-http/src/routes/tasks.rs @@ -1,49 +1,172 @@ use actix_web::{web, HttpRequest, HttpResponse}; -use meilisearch_error::ResponseError; -use meilisearch_lib::tasks::task::TaskId; +use meilisearch_lib::tasks::task::{TaskContent, TaskEvent, TaskId}; use meilisearch_lib::tasks::TaskFilter; use meilisearch_lib::MeiliSearch; +use meilisearch_types::error::ResponseError; +use meilisearch_types::index_uid::IndexUid; +use meilisearch_types::star_or::StarOr; +use serde::Deserialize; +use serde_cs::vec::CS; use serde_json::json; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::sequential_extractor::SeqHandler; -use crate::task::{TaskListView, TaskView}; +use crate::task::{TaskListView, TaskStatus, TaskType, TaskView}; + +use super::fold_star_or; + +const DEFAULT_LIMIT: fn() -> usize = || 20; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::get().to(SeqHandler(get_tasks)))) .service(web::resource("/{task_id}").route(web::get().to(SeqHandler(get_task)))); } +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TasksFilterQuery { + #[serde(rename = "type")] + type_: Option>>, + status: Option>>, + index_uid: Option>>, + #[serde(default = "DEFAULT_LIMIT")] + limit: usize, + from: Option, +} + +#[rustfmt::skip] +fn task_type_matches_content(type_: &TaskType, content: &TaskContent) -> bool { + matches!((type_, content), + (TaskType::IndexCreation, TaskContent::IndexCreation { .. }) + | (TaskType::IndexUpdate, TaskContent::IndexUpdate { .. }) + | (TaskType::IndexDeletion, TaskContent::IndexDeletion { .. }) + | (TaskType::DocumentAdditionOrUpdate, TaskContent::DocumentAddition { .. }) + | (TaskType::DocumentDeletion, TaskContent::DocumentDeletion{ .. }) + | (TaskType::SettingsUpdate, TaskContent::SettingsUpdate { .. }) + ) +} + +#[rustfmt::skip] +fn task_status_matches_events(status: &TaskStatus, events: &[TaskEvent]) -> bool { + events.last().map_or(false, |event| { + matches!((status, event), + (TaskStatus::Enqueued, TaskEvent::Created(_)) + | (TaskStatus::Processing, TaskEvent::Processing(_) | TaskEvent::Batched { .. }) + | (TaskStatus::Succeeded, TaskEvent::Succeeded { .. }) + | (TaskStatus::Failed, TaskEvent::Failed { .. }), + ) + }) +} + async fn get_tasks( meilisearch: GuardedData, MeiliSearch>, + params: web::Query, req: HttpRequest, analytics: web::Data, ) -> Result { + let TasksFilterQuery { + type_, + status, + index_uid, + limit, + from, + } = params.into_inner(); + + let search_rules = &meilisearch.filters().search_rules; + + // We first transform a potential indexUid=* into a "not specified indexUid filter" + // for every one of the filters: type, status, and indexUid. + let type_: Option> = type_.and_then(fold_star_or); + let status: Option> = status.and_then(fold_star_or); + let index_uid: Option> = index_uid.and_then(fold_star_or); + analytics.publish( "Tasks Seen".to_string(), - json!({ "per_task_uid": false }), + json!({ + "filtered_by_index_uid": index_uid.as_ref().map_or(false, |v| !v.is_empty()), + "filtered_by_type": type_.as_ref().map_or(false, |v| !v.is_empty()), + "filtered_by_status": status.as_ref().map_or(false, |v| !v.is_empty()), + }), Some(&req), ); - let search_rules = &meilisearch.filters().search_rules; - let filters = if search_rules.is_index_authorized("*") { - None - } else { - let mut filters = TaskFilter::default(); - for (index, _policy) in search_rules.clone() { - filters.filter_index(index); + // Then we filter on potential indexes and make sure that the search filter + // restrictions are also applied. + let indexes_filters = match index_uid { + Some(indexes) => { + let mut filters = TaskFilter::default(); + for name in indexes { + if search_rules.is_index_authorized(&name) { + filters.filter_index(name.to_string()); + } + } + Some(filters) + } + None => { + if search_rules.is_index_authorized("*") { + None + } else { + let mut filters = TaskFilter::default(); + for (index, _policy) in search_rules.clone() { + filters.filter_index(index); + } + Some(filters) + } } - Some(filters) }; - let tasks: TaskListView = meilisearch - .list_tasks(filters, None, None) + // Then we complete the task filter with other potential status and types filters. + let filters = if type_.is_some() || status.is_some() { + let mut filters = indexes_filters.unwrap_or_default(); + filters.filter_fn(move |task| { + let matches_type = match &type_ { + Some(types) => types + .iter() + .any(|t| task_type_matches_content(t, &task.content)), + None => true, + }; + + let matches_status = match &status { + Some(statuses) => statuses + .iter() + .any(|t| task_status_matches_events(t, &task.events)), + None => true, + }; + + matches_type && matches_status + }); + Some(filters) + } else { + indexes_filters + }; + + // We +1 just to know if there is more after this "page" or not. + let limit = limit.saturating_add(1); + + let mut tasks_results: Vec<_> = meilisearch + .list_tasks(filters, Some(limit), from) .await? .into_iter() .map(TaskView::from) - .collect::>() - .into(); + .collect(); + + // If we were able to fetch the number +1 tasks we asked + // it means that there is more to come. + let next = if tasks_results.len() == limit { + tasks_results.pop().map(|t| t.uid) + } else { + None + }; + + let from = tasks_results.first().map(|t| t.uid); + + let tasks = TaskListView { + results: tasks_results, + limit: limit.saturating_sub(1), + from, + next, + }; Ok(HttpResponse::Ok().json(tasks)) } diff --git a/meilisearch-http/src/task.rs b/meilisearch-http/src/task.rs index 7179b10db..06bba1f76 100644 --- a/meilisearch-http/src/task.rs +++ b/meilisearch-http/src/task.rs @@ -1,62 +1,137 @@ -use std::fmt::Write; +use std::error::Error; +use std::fmt::{self, Write}; +use std::str::FromStr; use std::write; -use meilisearch_error::ResponseError; use meilisearch_lib::index::{Settings, Unchecked}; -use meilisearch_lib::milli::update::IndexDocumentsMethod; use meilisearch_lib::tasks::batch::BatchId; use meilisearch_lib::tasks::task::{ DocumentDeletion, Task, TaskContent, TaskEvent, TaskId, TaskResult, }; -use serde::{Serialize, Serializer}; +use meilisearch_types::error::ResponseError; +use serde::{Deserialize, Serialize, Serializer}; use time::{Duration, OffsetDateTime}; use crate::AUTOBATCHING_ENABLED; -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -enum TaskType { +pub enum TaskType { IndexCreation, IndexUpdate, IndexDeletion, - DocumentAddition, - DocumentPartial, + DocumentAdditionOrUpdate, DocumentDeletion, SettingsUpdate, - ClearAll, + DumpCreation, } impl From for TaskType { fn from(other: TaskContent) -> Self { match other { - TaskContent::DocumentAddition { - merge_strategy: IndexDocumentsMethod::ReplaceDocuments, - .. - } => TaskType::DocumentAddition, - TaskContent::DocumentAddition { - merge_strategy: IndexDocumentsMethod::UpdateDocuments, - .. - } => TaskType::DocumentPartial, - TaskContent::DocumentDeletion(DocumentDeletion::Clear) => TaskType::ClearAll, - TaskContent::DocumentDeletion(DocumentDeletion::Ids(_)) => TaskType::DocumentDeletion, - TaskContent::SettingsUpdate { .. } => TaskType::SettingsUpdate, - TaskContent::IndexDeletion => TaskType::IndexDeletion, TaskContent::IndexCreation { .. } => TaskType::IndexCreation, TaskContent::IndexUpdate { .. } => TaskType::IndexUpdate, - _ => unreachable!("unexpected task type"), + TaskContent::IndexDeletion { .. } => TaskType::IndexDeletion, + TaskContent::DocumentAddition { .. } => TaskType::DocumentAdditionOrUpdate, + TaskContent::DocumentDeletion { .. } => TaskType::DocumentDeletion, + TaskContent::SettingsUpdate { .. } => TaskType::SettingsUpdate, + TaskContent::Dump { .. } => TaskType::DumpCreation, } } } -#[derive(Debug, Serialize)] +#[derive(Debug)] +pub struct TaskTypeError { + invalid_type: String, +} + +impl fmt::Display for TaskTypeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "invalid task type `{}`, expecting one of: \ + indexCreation, indexUpdate, indexDeletion, documentAdditionOrUpdate, \ + documentDeletion, settingsUpdate, dumpCreation", + self.invalid_type + ) + } +} + +impl Error for TaskTypeError {} + +impl FromStr for TaskType { + type Err = TaskTypeError; + + fn from_str(type_: &str) -> Result { + if type_.eq_ignore_ascii_case("indexCreation") { + Ok(TaskType::IndexCreation) + } else if type_.eq_ignore_ascii_case("indexUpdate") { + Ok(TaskType::IndexUpdate) + } else if type_.eq_ignore_ascii_case("indexDeletion") { + Ok(TaskType::IndexDeletion) + } else if type_.eq_ignore_ascii_case("documentAdditionOrUpdate") { + Ok(TaskType::DocumentAdditionOrUpdate) + } else if type_.eq_ignore_ascii_case("documentDeletion") { + Ok(TaskType::DocumentDeletion) + } else if type_.eq_ignore_ascii_case("settingsUpdate") { + Ok(TaskType::SettingsUpdate) + } else if type_.eq_ignore_ascii_case("dumpCreation") { + Ok(TaskType::DumpCreation) + } else { + Err(TaskTypeError { + invalid_type: type_.to_string(), + }) + } + } +} + +#[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -enum TaskStatus { +pub enum TaskStatus { Enqueued, Processing, Succeeded, Failed, } +#[derive(Debug)] +pub struct TaskStatusError { + invalid_status: String, +} + +impl fmt::Display for TaskStatusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "invalid task status `{}`, expecting one of: \ + enqueued, processing, succeeded, or failed", + self.invalid_status, + ) + } +} + +impl Error for TaskStatusError {} + +impl FromStr for TaskStatus { + type Err = TaskStatusError; + + fn from_str(status: &str) -> Result { + if status.eq_ignore_ascii_case("enqueued") { + Ok(TaskStatus::Enqueued) + } else if status.eq_ignore_ascii_case("processing") { + Ok(TaskStatus::Processing) + } else if status.eq_ignore_ascii_case("succeeded") { + Ok(TaskStatus::Succeeded) + } else if status.eq_ignore_ascii_case("failed") { + Ok(TaskStatus::Failed) + } else { + Err(TaskStatusError { + invalid_status: status.to_string(), + }) + } + } +} + #[derive(Debug, Serialize)] #[serde(untagged)] #[allow(clippy::large_enum_variant)] @@ -80,6 +155,8 @@ enum TaskDetails { }, #[serde(rename_all = "camelCase")] ClearAll { deleted_documents: Option }, + #[serde(rename_all = "camelCase")] + Dump { dump_uid: String }, } /// Serialize a `time::Duration` as a best effort ISO 8601 while waiting for @@ -136,8 +213,8 @@ fn serialize_duration( #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct TaskView { - uid: TaskId, - index_uid: String, + pub uid: TaskId, + index_uid: Option, status: TaskStatus, #[serde(rename = "type")] task_type: TaskType, @@ -159,46 +236,44 @@ pub struct TaskView { impl From for TaskView { fn from(task: Task) -> Self { + let index_uid = task.index_uid().map(String::from); let Task { id, - index_uid, content, events, } = task; let (task_type, mut details) = match content { TaskContent::DocumentAddition { - merge_strategy, - documents_count, - .. + documents_count, .. } => { let details = TaskDetails::DocumentAddition { received_documents: documents_count, indexed_documents: None, }; - let task_type = match merge_strategy { - IndexDocumentsMethod::UpdateDocuments => TaskType::DocumentPartial, - IndexDocumentsMethod::ReplaceDocuments => TaskType::DocumentAddition, - _ => unreachable!("Unexpected document merge strategy."), - }; - - (task_type, Some(details)) + (TaskType::DocumentAdditionOrUpdate, Some(details)) } - TaskContent::DocumentDeletion(DocumentDeletion::Ids(ids)) => ( + TaskContent::DocumentDeletion { + deletion: DocumentDeletion::Ids(ids), + .. + } => ( TaskType::DocumentDeletion, Some(TaskDetails::DocumentDeletion { received_document_ids: ids.len(), deleted_documents: None, }), ), - TaskContent::DocumentDeletion(DocumentDeletion::Clear) => ( - TaskType::ClearAll, + TaskContent::DocumentDeletion { + deletion: DocumentDeletion::Clear, + .. + } => ( + TaskType::DocumentDeletion, Some(TaskDetails::ClearAll { deleted_documents: None, }), ), - TaskContent::IndexDeletion => ( + TaskContent::IndexDeletion { .. } => ( TaskType::IndexDeletion, Some(TaskDetails::ClearAll { deleted_documents: None, @@ -208,14 +283,18 @@ impl From for TaskView { TaskType::SettingsUpdate, Some(TaskDetails::Settings { settings }), ), - TaskContent::IndexCreation { primary_key } => ( + TaskContent::IndexCreation { primary_key, .. } => ( TaskType::IndexCreation, Some(TaskDetails::IndexInfo { primary_key }), ), - TaskContent::IndexUpdate { primary_key } => ( + TaskContent::IndexUpdate { primary_key, .. } => ( TaskType::IndexUpdate, Some(TaskDetails::IndexInfo { primary_key }), ), + TaskContent::Dump { uid } => ( + TaskType::DumpCreation, + Some(TaskDetails::Dump { dump_uid: uid }), + ), }; // An event always has at least one event: "Created" @@ -223,7 +302,7 @@ impl From for TaskView { TaskEvent::Created(_) => (TaskStatus::Enqueued, None, None), TaskEvent::Batched { .. } => (TaskStatus::Enqueued, None, None), TaskEvent::Processing(_) => (TaskStatus::Processing, None, None), - TaskEvent::Succeded { timestamp, result } => { + TaskEvent::Succeeded { timestamp, result } => { match (result, &mut details) { ( TaskResult::DocumentAddition { @@ -313,7 +392,7 @@ impl From for TaskView { Self { uid: id, - index_uid: index_uid.into_inner(), + index_uid, status, task_type, details, @@ -329,20 +408,17 @@ impl From for TaskView { #[derive(Debug, Serialize)] pub struct TaskListView { - results: Vec, -} - -impl From> for TaskListView { - fn from(results: Vec) -> Self { - Self { results } - } + pub results: Vec, + pub limit: usize, + pub from: Option, + pub next: Option, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct SummarizedTaskView { - uid: TaskId, - index_uid: String, + task_uid: TaskId, + index_uid: Option, status: TaskStatus, #[serde(rename = "type")] task_type: TaskType, @@ -364,8 +440,8 @@ impl From for SummarizedTaskView { }; Self { - uid: other.id, - index_uid: other.index_uid.to_string(), + task_uid: other.id, + index_uid: other.index_uid().map(String::from), status: TaskStatus::Enqueued, task_type: other.content.into(), enqueued_at, diff --git a/meilisearch-http/tests/assets/v1_v0.20.0_movies.dump b/meilisearch-http/tests/assets/v1_v0.20.0_movies.dump new file mode 100644 index 000000000..9d0f4e066 Binary files /dev/null and b/meilisearch-http/tests/assets/v1_v0.20.0_movies.dump differ diff --git a/meilisearch-http/tests/assets/v1_v0.20.0_movies_with_settings.dump b/meilisearch-http/tests/assets/v1_v0.20.0_movies_with_settings.dump new file mode 100644 index 000000000..5f2096e58 Binary files /dev/null and b/meilisearch-http/tests/assets/v1_v0.20.0_movies_with_settings.dump differ diff --git a/meilisearch-http/tests/assets/v1_v0.20.0_rubygems_with_settings.dump b/meilisearch-http/tests/assets/v1_v0.20.0_rubygems_with_settings.dump new file mode 100644 index 000000000..83436d4e7 Binary files /dev/null and b/meilisearch-http/tests/assets/v1_v0.20.0_rubygems_with_settings.dump differ diff --git a/meilisearch-http/tests/assets/v2_v0.21.1_movies.dump b/meilisearch-http/tests/assets/v2_v0.21.1_movies.dump new file mode 100644 index 000000000..ee7bf0600 Binary files /dev/null and b/meilisearch-http/tests/assets/v2_v0.21.1_movies.dump differ diff --git a/meilisearch-http/tests/assets/v2_v0.21.1_movies_with_settings.dump b/meilisearch-http/tests/assets/v2_v0.21.1_movies_with_settings.dump new file mode 100644 index 000000000..03483a264 Binary files /dev/null and b/meilisearch-http/tests/assets/v2_v0.21.1_movies_with_settings.dump differ diff --git a/meilisearch-http/tests/assets/v2_v0.21.1_rubygems_with_settings.dump b/meilisearch-http/tests/assets/v2_v0.21.1_rubygems_with_settings.dump new file mode 100644 index 000000000..1bad10b87 Binary files /dev/null and b/meilisearch-http/tests/assets/v2_v0.21.1_rubygems_with_settings.dump differ diff --git a/meilisearch-http/tests/assets/v3_v0.24.0_movies.dump b/meilisearch-http/tests/assets/v3_v0.24.0_movies.dump new file mode 100644 index 000000000..3d692812b Binary files /dev/null and b/meilisearch-http/tests/assets/v3_v0.24.0_movies.dump differ diff --git a/meilisearch-http/tests/assets/v3_v0.24.0_movies_with_settings.dump b/meilisearch-http/tests/assets/v3_v0.24.0_movies_with_settings.dump new file mode 100644 index 000000000..5b9990078 Binary files /dev/null and b/meilisearch-http/tests/assets/v3_v0.24.0_movies_with_settings.dump differ diff --git a/meilisearch-http/tests/assets/v3_v0.24.0_rubygems_with_settings.dump b/meilisearch-http/tests/assets/v3_v0.24.0_rubygems_with_settings.dump new file mode 100644 index 000000000..7f7baeab5 Binary files /dev/null and b/meilisearch-http/tests/assets/v3_v0.24.0_rubygems_with_settings.dump differ diff --git a/meilisearch-http/tests/assets/v4_v0.25.2_movies.dump b/meilisearch-http/tests/assets/v4_v0.25.2_movies.dump new file mode 100644 index 000000000..7e063dc07 Binary files /dev/null and b/meilisearch-http/tests/assets/v4_v0.25.2_movies.dump differ diff --git a/meilisearch-http/tests/assets/v4_v0.25.2_movies_with_settings.dump b/meilisearch-http/tests/assets/v4_v0.25.2_movies_with_settings.dump new file mode 100644 index 000000000..4374e131c Binary files /dev/null and b/meilisearch-http/tests/assets/v4_v0.25.2_movies_with_settings.dump differ diff --git a/meilisearch-http/tests/assets/v4_v0.25.2_rubygems_with_settings.dump b/meilisearch-http/tests/assets/v4_v0.25.2_rubygems_with_settings.dump new file mode 100644 index 000000000..1dfc22e8e Binary files /dev/null and b/meilisearch-http/tests/assets/v4_v0.25.2_rubygems_with_settings.dump differ diff --git a/meilisearch-http/tests/assets/v5_v0.28.0_test_dump.dump b/meilisearch-http/tests/assets/v5_v0.28.0_test_dump.dump new file mode 100644 index 000000000..bc30ea7b9 Binary files /dev/null and b/meilisearch-http/tests/assets/v5_v0.28.0_test_dump.dump differ diff --git a/meilisearch-http/tests/auth/api_keys.rs b/meilisearch-http/tests/auth/api_keys.rs index e9fb3d127..7fdf2f129 100644 --- a/meilisearch-http/tests/auth/api_keys.rs +++ b/meilisearch-http/tests/auth/api_keys.rs @@ -9,7 +9,9 @@ async fn add_valid_api_key() { server.use_api_key("MASTER_KEY"); let content = json!({ + "name": "indexing-key", "description": "Indexing API key", + "uid": "4bc0887a-0e41-4f3b-935d-0c451dcee9c8", "indexes": ["products"], "actions": [ "search", @@ -25,19 +27,22 @@ async fn add_valid_api_key() { "settings.update", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["expiresAt"].is_string()); assert!(response["createdAt"].is_string()); assert!(response["updatedAt"].is_string()); let expected_response = json!({ + "name": "indexing-key", "description": "Indexing API key", + "uid": "4bc0887a-0e41-4f3b-935d-0c451dcee9c8", + "key": "d9e776b8412f1db6974c9a5556b961c3559440b6588216f4ea5d9ed49f7c8f3c", "indexes": ["products"], "actions": [ "search", @@ -53,13 +58,11 @@ async fn add_valid_api_key() { "settings.update", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); assert_json_include!(actual: response, expected: expected_response); - assert_eq!(code, 201); } #[actix_rt::test] @@ -84,13 +87,13 @@ async fn add_valid_api_key_expired_at() { "settings.update", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13" }); let (response, code) = server.add_api_key(content).await; - assert!(response["key"].is_string(), "{:?}", response); + assert_eq!(201, code, "{:?}", &response); + assert!(response["key"].is_string()); assert!(response["expiresAt"].is_string()); assert!(response["createdAt"].is_string()); assert!(response["updatedAt"].is_string()); @@ -112,13 +115,11 @@ async fn add_valid_api_key_expired_at() { "settings.update", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); assert_json_include!(actual: response, expected: expected_response); - assert_eq!(code, 201); } #[actix_rt::test] @@ -128,23 +129,19 @@ async fn add_valid_api_key_no_description() { let content = json!({ "indexes": ["products"], - "actions": [ - "documents.add" - ], + "actions": ["documents.add"], "expiresAt": "2050-11-13T00:00:00" }); let (response, code) = server.add_api_key(content).await; - + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["expiresAt"].is_string()); assert!(response["createdAt"].is_string()); assert!(response["updatedAt"].is_string()); let expected_response = json!({ - "actions": [ - "documents.add" - ], + "actions": ["documents.add"], "indexes": [ "products" ], @@ -152,7 +149,6 @@ async fn add_valid_api_key_no_description() { }); assert_json_include!(actual: response, expected: expected_response); - assert_eq!(code, 201); } #[actix_rt::test] @@ -163,23 +159,19 @@ async fn add_valid_api_key_null_description() { let content = json!({ "description": Value::Null, "indexes": ["products"], - "actions": [ - "documents.add" - ], + "actions": ["documents.add"], "expiresAt": "2050-11-13T00:00:00" }); let (response, code) = server.add_api_key(content).await; - + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["expiresAt"].is_string()); assert!(response["createdAt"].is_string()); assert!(response["updatedAt"].is_string()); let expected_response = json!({ - "actions": [ - "documents.add" - ], + "actions": ["documents.add"], "indexes": [ "products" ], @@ -187,7 +179,6 @@ async fn add_valid_api_key_null_description() { }); assert_json_include!(actual: response, expected: expected_response); - assert_eq!(code, 201); } #[actix_rt::test] @@ -196,12 +187,11 @@ async fn error_add_api_key_no_header() { let content = json!({ "description": "Indexing API key", "indexes": ["products"], - "actions": [ - "documents.add" - ], + "actions": ["documents.add"], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; + assert_eq!(401, code, "{:?}", &response); let expected_response = json!({ "message": "The Authorization header is missing. It must use the bearer authorization method.", @@ -211,7 +201,6 @@ async fn error_add_api_key_no_header() { }); assert_eq!(response, expected_response); - assert_eq!(code, 401); } #[actix_rt::test] @@ -222,12 +211,11 @@ async fn error_add_api_key_bad_key() { let content = json!({ "description": "Indexing API key", "indexes": ["products"], - "actions": [ - "documents.add" - ], + "actions": ["documents.add"], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; + assert_eq!(403, code, "{:?}", &response); let expected_response = json!({ "message": "The provided API key is invalid.", @@ -237,7 +225,6 @@ async fn error_add_api_key_bad_key() { }); assert_eq!(response, expected_response); - assert_eq!(code, 403); } #[actix_rt::test] @@ -252,6 +239,7 @@ async fn error_add_api_key_missing_parameter() { "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ "message": "`indexes` field is mandatory.", @@ -261,7 +249,6 @@ async fn error_add_api_key_missing_parameter() { }); assert_eq!(response, expected_response); - assert_eq!(code, 400); // missing actions let content = json!({ @@ -270,6 +257,7 @@ async fn error_add_api_key_missing_parameter() { "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ "message": "`actions` field is mandatory.", @@ -279,7 +267,6 @@ async fn error_add_api_key_missing_parameter() { }); assert_eq!(response, expected_response); - assert_eq!(code, 400); // missing expiration date let content = json!({ @@ -288,6 +275,7 @@ async fn error_add_api_key_missing_parameter() { "actions": ["documents.add"], }); let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ "message": "`expiresAt` field is mandatory.", @@ -297,7 +285,6 @@ async fn error_add_api_key_missing_parameter() { }); assert_eq!(response, expected_response); - assert_eq!(code, 400); } #[actix_rt::test] @@ -308,12 +295,11 @@ async fn error_add_api_key_invalid_parameters_description() { let content = json!({ "description": {"name":"products"}, "indexes": ["products"], - "actions": [ - "documents.add" - ], + "actions": ["documents.add"], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ "message": r#"`description` field value `{"name":"products"}` is invalid. It should be a string or specified as a null value."#, @@ -323,7 +309,30 @@ async fn error_add_api_key_invalid_parameters_description() { }); assert_eq!(response, expected_response); - assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn error_add_api_key_invalid_parameters_name() { + let mut server = Server::new_auth().await; + server.use_api_key("MASTER_KEY"); + + let content = json!({ + "name": {"name":"products"}, + "indexes": ["products"], + "actions": ["documents.add"], + "expiresAt": "2050-11-13T00:00:00Z" + }); + let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); + + let expected_response = json!({ + "message": r#"`name` field value `{"name":"products"}` is invalid. It should be a string or specified as a null value."#, + "code": "invalid_api_key_name", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_api_key_name" + }); + + assert_eq!(response, expected_response); } #[actix_rt::test] @@ -334,15 +343,39 @@ async fn error_add_api_key_invalid_parameters_indexes() { let content = json!({ "description": "Indexing API key", "indexes": {"name":"products"}, + "actions": ["documents.add"], + "expiresAt": "2050-11-13T00:00:00Z" + }); + let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); + + let expected_response = json!({ + "message": r#"`indexes` field value `{"name":"products"}` is invalid. It should be an array of string representing index names."#, + "code": "invalid_api_key_indexes", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_api_key_indexes" + }); + + assert_eq!(response, expected_response); +} + +#[actix_rt::test] +async fn error_add_api_key_invalid_index_uids() { + let mut server = Server::new_auth().await; + server.use_api_key("MASTER_KEY"); + + let content = json!({ + "description": Value::Null, + "indexes": ["invalid index # / \\name with spaces"], "actions": [ "documents.add" ], - "expiresAt": "2050-11-13T00:00:00Z" + "expiresAt": "2050-11-13T00:00:00" }); let (response, code) = server.add_api_key(content).await; let expected_response = json!({ - "message": r#"`indexes` field value `{"name":"products"}` is invalid. It should be an array of string representing index names."#, + "message": r#"`indexes` field value `["invalid index # / \\name with spaces"]` is invalid. It should be an array of string representing index names."#, "code": "invalid_api_key_indexes", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_indexes" @@ -364,6 +397,7 @@ async fn error_add_api_key_invalid_parameters_actions() { "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ "message": r#"`actions` field value `{"name":"products"}` is invalid. It should be an array of string representing action names."#, @@ -373,7 +407,6 @@ async fn error_add_api_key_invalid_parameters_actions() { }); assert_eq!(response, expected_response); - assert_eq!(code, 400); let content = json!({ "description": "Indexing API key", @@ -384,6 +417,7 @@ async fn error_add_api_key_invalid_parameters_actions() { "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ "message": r#"`actions` field value `["doc.add"]` is invalid. It should be an array of string representing action names."#, @@ -393,7 +427,6 @@ async fn error_add_api_key_invalid_parameters_actions() { }); assert_eq!(response, expected_response); - assert_eq!(code, 400); } #[actix_rt::test] @@ -404,12 +437,11 @@ async fn error_add_api_key_invalid_parameters_expires_at() { let content = json!({ "description": "Indexing API key", "indexes": ["products"], - "actions": [ - "documents.add" - ], + "actions": ["documents.add"], "expiresAt": {"name":"products"} }); let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ "message": r#"`expiresAt` field value `{"name":"products"}` is invalid. It should follow the RFC 3339 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'."#, @@ -419,7 +451,6 @@ async fn error_add_api_key_invalid_parameters_expires_at() { }); assert_eq!(response, expected_response); - assert_eq!(code, 400); } #[actix_rt::test] @@ -430,12 +461,11 @@ async fn error_add_api_key_invalid_parameters_expires_at_in_the_past() { let content = json!({ "description": "Indexing API key", "indexes": ["products"], - "actions": [ - "documents.add" - ], + "actions": ["documents.add"], "expiresAt": "2010-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ "message": r#"`expiresAt` field value `"2010-11-13T00:00:00Z"` is invalid. It should follow the RFC 3339 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'."#, @@ -445,7 +475,60 @@ async fn error_add_api_key_invalid_parameters_expires_at_in_the_past() { }); assert_eq!(response, expected_response); - assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn error_add_api_key_invalid_parameters_uid() { + let mut server = Server::new_auth().await; + server.use_api_key("MASTER_KEY"); + + let content = json!({ + "description": "Indexing API key", + "uid": "aaaaabbbbbccc", + "indexes": ["products"], + "actions": ["documents.add"], + "expiresAt": "2050-11-13T00:00:00Z" + }); + let (response, code) = server.add_api_key(content).await; + assert_eq!(400, code, "{:?}", &response); + + let expected_response = json!({ + "message": r#"`uid` field value `"aaaaabbbbbccc"` is invalid. It should be a valid UUID v4 string or omitted."#, + "code": "invalid_api_key_uid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_api_key_uid" + }); + + assert_eq!(response, expected_response); +} + +#[actix_rt::test] +async fn error_add_api_key_parameters_uid_already_exist() { + let mut server = Server::new_auth().await; + server.use_api_key("MASTER_KEY"); + let content = json!({ + "uid": "4bc0887a-0e41-4f3b-935d-0c451dcee9c8", + "indexes": ["products"], + "actions": ["search"], + "expiresAt": "2050-11-13T00:00:00Z" + }); + + // first creation is valid. + let (response, code) = server.add_api_key(content.clone()).await; + assert_eq!(201, code, "{:?}", &response); + + // uid already exist. + let (response, code) = server.add_api_key(content).await; + assert_eq!(409, code, "{:?}", &response); + + let expected_response = json!({ + "message": "`uid` field value `4bc0887a-0e41-4f3b-935d-0c451dcee9c8` is already an existing API key.", + "code": "api_key_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#api_key_already_exists" + }); + + assert_eq!(response, expected_response); } #[actix_rt::test] @@ -453,9 +536,11 @@ async fn get_api_key() { let mut server = Server::new_auth().await; server.use_api_key("MASTER_KEY"); + let uid = "4bc0887a-0e41-4f3b-935d-0c451dcee9c8"; let content = json!({ "description": "Indexing API key", "indexes": ["products"], + "uid": uid.to_string(), "actions": [ "search", "documents.add", @@ -470,27 +555,21 @@ async fn get_api_key() { "settings.update", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; // must pass if add_valid_api_key test passes. - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); let key = response["key"].as_str().unwrap(); - let (response, code) = server.get_api_key(&key).await; - assert!(response["key"].is_string()); - assert!(response["expiresAt"].is_string()); - assert!(response["createdAt"].is_string()); - assert!(response["updatedAt"].is_string()); - let expected_response = json!({ "description": "Indexing API key", "indexes": ["products"], + "uid": uid.to_string(), "actions": [ "search", "documents.add", @@ -505,13 +584,27 @@ async fn get_api_key() { "settings.update", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); - assert_json_include!(actual: response, expected: expected_response); - assert_eq!(code, 200); + // get with uid + let (response, code) = server.get_api_key(&uid).await; + assert_eq!(200, code, "{:?}", &response); + assert!(response["key"].is_string()); + assert!(response["expiresAt"].is_string()); + assert!(response["createdAt"].is_string()); + assert!(response["updatedAt"].is_string()); + assert_json_include!(actual: response, expected: &expected_response); + + // get with key + let (response, code) = server.get_api_key(&key).await; + assert_eq!(200, code, "{:?}", &response); + assert!(response["key"].is_string()); + assert!(response["expiresAt"].is_string()); + assert!(response["createdAt"].is_string()); + assert!(response["updatedAt"].is_string()); + assert_json_include!(actual: response, expected: &expected_response); } #[actix_rt::test] @@ -521,6 +614,7 @@ async fn error_get_api_key_no_header() { let (response, code) = server .get_api_key("d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4") .await; + assert_eq!(401, code, "{:?}", &response); let expected_response = json!({ "message": "The Authorization header is missing. It must use the bearer authorization method.", @@ -530,7 +624,6 @@ async fn error_get_api_key_no_header() { }); assert_eq!(response, expected_response); - assert_eq!(code, 401); } #[actix_rt::test] @@ -541,6 +634,7 @@ async fn error_get_api_key_bad_key() { let (response, code) = server .get_api_key("d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4") .await; + assert_eq!(403, code, "{:?}", &response); let expected_response = json!({ "message": "The provided API key is invalid.", @@ -550,7 +644,6 @@ async fn error_get_api_key_bad_key() { }); assert_eq!(response, expected_response); - assert_eq!(code, 403); } #[actix_rt::test] @@ -561,6 +654,7 @@ async fn error_get_api_key_not_found() { let (response, code) = server .get_api_key("d0552b41d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4") .await; + assert_eq!(404, code, "{:?}", &response); let expected_response = json!({ "message": "API key `d0552b41d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4` not found.", @@ -570,7 +664,6 @@ async fn error_get_api_key_not_found() { }); assert_eq!(response, expected_response); - assert_eq!(code, 404); } #[actix_rt::test] @@ -595,56 +688,60 @@ async fn list_api_keys() { "settings.update", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); - let (_response, code) = server.add_api_key(content).await; + let (response, code) = server.add_api_key(content).await; // must pass if add_valid_api_key test passes. - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); let (response, code) = server.list_api_keys().await; + assert_eq!(200, code, "{:?}", &response); let expected_response = json!({ "results": - [ - { - "description": "Indexing API key", - "indexes": ["products"], - "actions": [ - "search", - "documents.add", - "documents.get", - "documents.delete", - "indexes.create", - "indexes.get", - "indexes.update", - "indexes.delete", - "tasks.get", - "settings.get", - "settings.update", - "stats.get", - "dumps.create", - "dumps.get" - ], - "expiresAt": "2050-11-13T00:00:00Z" - }, - { - "description": "Default Search API Key (Use it to search from the frontend)", - "indexes": ["*"], - "actions": ["search"], - "expiresAt": serde_json::Value::Null, - }, - { - "description": "Default Admin API Key (Use it for all other operations. Caution! Do not use it on a public frontend)", - "indexes": ["*"], - "actions": ["*"], - "expiresAt": serde_json::Value::Null, - } - ]}); + [ + { + "description": "Indexing API key", + "indexes": ["products"], + "actions": [ + "search", + "documents.add", + "documents.get", + "documents.delete", + "indexes.create", + "indexes.get", + "indexes.update", + "indexes.delete", + "tasks.get", + "settings.get", + "settings.update", + "stats.get", + "dumps.create", + ], + "expiresAt": "2050-11-13T00:00:00Z" + }, + { + "name": "Default Search API Key", + "description": "Use it to search from the frontend", + "indexes": ["*"], + "actions": ["search"], + "expiresAt": serde_json::Value::Null, + }, + { + "name": "Default Admin API Key", + "description": "Use it for anything that is not a search operation. Caution! Do not expose it on a public frontend", + "indexes": ["*"], + "actions": ["*"], + "expiresAt": serde_json::Value::Null, + } + ], + "limit": 20, + "offset": 0, + "total": 3, + }); assert_json_include!(actual: response, expected: expected_response); - assert_eq!(code, 200); } #[actix_rt::test] @@ -652,6 +749,7 @@ async fn error_list_api_keys_no_header() { let server = Server::new_auth().await; let (response, code) = server.list_api_keys().await; + assert_eq!(401, code, "{:?}", &response); let expected_response = json!({ "message": "The Authorization header is missing. It must use the bearer authorization method.", @@ -661,7 +759,6 @@ async fn error_list_api_keys_no_header() { }); assert_eq!(response, expected_response); - assert_eq!(code, 401); } #[actix_rt::test] @@ -670,6 +767,7 @@ async fn error_list_api_keys_bad_key() { server.use_api_key("d4000bd7225f77d1eb22cc706ed36772bbc36767c016a27f76def7537b68600d"); let (response, code) = server.list_api_keys().await; + assert_eq!(403, code, "{:?}", &response); let expected_response = json!({ "message": "The provided API key is invalid.", @@ -679,7 +777,6 @@ async fn error_list_api_keys_bad_key() { }); assert_eq!(response, expected_response); - assert_eq!(code, 403); } #[actix_rt::test] @@ -704,24 +801,23 @@ async fn delete_api_key() { "settings.update", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; // must pass if add_valid_api_key test passes. - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); - let key = response["key"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); - let (_response, code) = server.delete_api_key(&key).await; - assert_eq!(code, 204); + let (response, code) = server.delete_api_key(&uid).await; + assert_eq!(204, code, "{:?}", &response); // check if API key no longer exist. - let (_response, code) = server.get_api_key(&key).await; - assert_eq!(code, 404); + let (response, code) = server.get_api_key(&uid).await; + assert_eq!(404, code, "{:?}", &response); } #[actix_rt::test] @@ -731,6 +827,7 @@ async fn error_delete_api_key_no_header() { let (response, code) = server .delete_api_key("d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4") .await; + assert_eq!(401, code, "{:?}", &response); let expected_response = json!({ "message": "The Authorization header is missing. It must use the bearer authorization method.", @@ -740,7 +837,6 @@ async fn error_delete_api_key_no_header() { }); assert_eq!(response, expected_response); - assert_eq!(code, 401); } #[actix_rt::test] @@ -751,6 +847,7 @@ async fn error_delete_api_key_bad_key() { let (response, code) = server .delete_api_key("d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4") .await; + assert_eq!(403, code, "{:?}", &response); let expected_response = json!({ "message": "The provided API key is invalid.", @@ -760,7 +857,6 @@ async fn error_delete_api_key_bad_key() { }); assert_eq!(response, expected_response); - assert_eq!(code, 403); } #[actix_rt::test] @@ -771,6 +867,7 @@ async fn error_delete_api_key_not_found() { let (response, code) = server .delete_api_key("d0552b41d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4") .await; + assert_eq!(404, code, "{:?}", &response); let expected_response = json!({ "message": "API key `d0552b41d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4` not found.", @@ -780,7 +877,6 @@ async fn error_delete_api_key_not_found() { }); assert_eq!(response, expected_response); - assert_eq!(code, 404); } #[actix_rt::test] @@ -801,19 +897,18 @@ async fn patch_api_key_description() { "indexes.delete", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; // must pass if add_valid_api_key test passes. - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["createdAt"].is_string()); assert!(response["updatedAt"].is_string()); - let key = response["key"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); let created_at = response["createdAt"].as_str().unwrap(); let updated_at = response["updatedAt"].as_str().unwrap(); @@ -821,7 +916,8 @@ async fn patch_api_key_description() { let content = json!({ "description": "Indexing API key" }); thread::sleep(time::Duration::new(1, 0)); - let (response, code) = server.patch_api_key(&key, content).await; + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(200, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["expiresAt"].is_string()); assert!(response["createdAt"].is_string()); @@ -842,24 +938,23 @@ async fn patch_api_key_description() { "indexes.delete", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); assert_json_include!(actual: response, expected: expected); - assert_eq!(code, 200); // Change the description - let content = json!({ "description": "Porduct API key" }); + let content = json!({ "description": "Product API key" }); - let (response, code) = server.patch_api_key(&key, content).await; + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(200, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["expiresAt"].is_string()); assert!(response["createdAt"].is_string()); let expected = json!({ - "description": "Porduct API key", + "description": "Product API key", "indexes": ["products"], "actions": [ "search", @@ -872,18 +967,17 @@ async fn patch_api_key_description() { "indexes.delete", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); assert_json_include!(actual: response, expected: expected); - assert_eq!(code, 200); // Remove the description let content = json!({ "description": serde_json::Value::Null }); - let (response, code) = server.patch_api_key(&key, content).await; + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(200, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["expiresAt"].is_string()); assert!(response["createdAt"].is_string()); @@ -901,17 +995,138 @@ async fn patch_api_key_description() { "indexes.delete", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); assert_json_include!(actual: response, expected: expected); - assert_eq!(code, 200); } #[actix_rt::test] -async fn patch_api_key_indexes() { +async fn patch_api_key_name() { + let mut server = Server::new_auth().await; + server.use_api_key("MASTER_KEY"); + + let content = json!({ + "indexes": ["products"], + "actions": [ + "search", + "documents.add", + "documents.get", + "documents.delete", + "indexes.create", + "indexes.get", + "indexes.update", + "indexes.delete", + "stats.get", + "dumps.create", + ], + "expiresAt": "2050-11-13T00:00:00Z" + }); + + let (response, code) = server.add_api_key(content).await; + // must pass if add_valid_api_key test passes. + assert_eq!(201, code, "{:?}", &response); + assert!(response["key"].is_string()); + assert!(response["createdAt"].is_string()); + assert!(response["updatedAt"].is_string()); + + let uid = response["uid"].as_str().unwrap(); + let created_at = response["createdAt"].as_str().unwrap(); + let updated_at = response["updatedAt"].as_str().unwrap(); + + // Add a name + let content = json!({ "name": "Indexing API key" }); + + thread::sleep(time::Duration::new(1, 0)); + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(200, code, "{:?}", &response); + assert!(response["key"].is_string()); + assert!(response["expiresAt"].is_string()); + assert!(response["createdAt"].is_string()); + assert_ne!(response["updatedAt"].as_str().unwrap(), updated_at); + assert_eq!(response["createdAt"].as_str().unwrap(), created_at); + + let expected = json!({ + "name": "Indexing API key", + "indexes": ["products"], + "actions": [ + "search", + "documents.add", + "documents.get", + "documents.delete", + "indexes.create", + "indexes.get", + "indexes.update", + "indexes.delete", + "stats.get", + "dumps.create", + ], + "expiresAt": "2050-11-13T00:00:00Z" + }); + + assert_json_include!(actual: response, expected: expected); + + // Change the name + let content = json!({ "name": "Product API key" }); + + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(200, code, "{:?}", &response); + assert!(response["key"].is_string()); + assert!(response["expiresAt"].is_string()); + assert!(response["createdAt"].is_string()); + + let expected = json!({ + "name": "Product API key", + "indexes": ["products"], + "actions": [ + "search", + "documents.add", + "documents.get", + "documents.delete", + "indexes.create", + "indexes.get", + "indexes.update", + "indexes.delete", + "stats.get", + "dumps.create", + ], + "expiresAt": "2050-11-13T00:00:00Z" + }); + + assert_json_include!(actual: response, expected: expected); + + // Remove the name + let content = json!({ "name": serde_json::Value::Null }); + + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(200, code, "{:?}", &response); + assert!(response["key"].is_string()); + assert!(response["expiresAt"].is_string()); + assert!(response["createdAt"].is_string()); + + let expected = json!({ + "indexes": ["products"], + "actions": [ + "search", + "documents.add", + "documents.get", + "documents.delete", + "indexes.create", + "indexes.get", + "indexes.update", + "indexes.delete", + "stats.get", + "dumps.create", + ], + "expiresAt": "2050-11-13T00:00:00Z" + }); + + assert_json_include!(actual: response, expected: expected); +} + +#[actix_rt::test] +async fn error_patch_api_key_indexes() { let mut server = Server::new_auth().await; server.use_api_key("MASTER_KEY"); @@ -929,57 +1144,36 @@ async fn patch_api_key_indexes() { "indexes.delete", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; // must pass if add_valid_api_key test passes. - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["createdAt"].is_string()); assert!(response["updatedAt"].is_string()); - let key = response["key"].as_str().unwrap(); - let created_at = response["createdAt"].as_str().unwrap(); - let updated_at = response["updatedAt"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); let content = json!({ "indexes": ["products", "prices"] }); thread::sleep(time::Duration::new(1, 0)); - let (response, code) = server.patch_api_key(&key, content).await; - assert!(response["key"].is_string()); - assert!(response["expiresAt"].is_string()); - assert!(response["createdAt"].is_string()); - assert_ne!(response["updatedAt"].as_str().unwrap(), updated_at); - assert_eq!(response["createdAt"].as_str().unwrap(), created_at); + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(400, code, "{:?}", &response); - let expected = json!({ - "description": "Indexing API key", - "indexes": ["products", "prices"], - "actions": [ - "search", - "documents.add", - "documents.get", - "documents.delete", - "indexes.create", - "indexes.get", - "indexes.update", - "indexes.delete", - "stats.get", - "dumps.create", - "dumps.get" - ], - "expiresAt": "2050-11-13T00:00:00Z" + let expected = json!({"message": "The `indexes` field cannot be modified for the given resource.", + "code": "immutable_field", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_field" }); assert_json_include!(actual: response, expected: expected); - assert_eq!(code, 200); } #[actix_rt::test] -async fn patch_api_key_actions() { +async fn error_patch_api_key_actions() { let mut server = Server::new_auth().await; server.use_api_key("MASTER_KEY"); @@ -997,21 +1191,18 @@ async fn patch_api_key_actions() { "indexes.delete", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; // must pass if add_valid_api_key test passes. - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["createdAt"].is_string()); assert!(response["updatedAt"].is_string()); - let key = response["key"].as_str().unwrap(); - let created_at = response["createdAt"].as_str().unwrap(); - let updated_at = response["updatedAt"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); let content = json!({ "actions": [ @@ -1024,32 +1215,20 @@ async fn patch_api_key_actions() { }); thread::sleep(time::Duration::new(1, 0)); - let (response, code) = server.patch_api_key(&key, content).await; - assert!(response["key"].is_string()); - assert!(response["expiresAt"].is_string()); - assert!(response["createdAt"].is_string()); - assert_ne!(response["updatedAt"].as_str().unwrap(), updated_at); - assert_eq!(response["createdAt"].as_str().unwrap(), created_at); + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(400, code, "{:?}", &response); - let expected = json!({ - "description": "Indexing API key", - "indexes": ["products"], - "actions": [ - "search", - "documents.get", - "indexes.get", - "tasks.get", - "settings.get", - ], - "expiresAt": "2050-11-13T00:00:00Z" + let expected = json!({"message": "The `actions` field cannot be modified for the given resource.", + "code": "immutable_field", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_field" }); assert_json_include!(actual: response, expected: expected); - assert_eq!(code, 200); } #[actix_rt::test] -async fn patch_api_key_expiration_date() { +async fn error_patch_api_key_expiration_date() { let mut server = Server::new_auth().await; server.use_api_key("MASTER_KEY"); @@ -1067,53 +1246,32 @@ async fn patch_api_key_expiration_date() { "indexes.delete", "stats.get", "dumps.create", - "dumps.get" ], "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; // must pass if add_valid_api_key test passes. - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); assert!(response["createdAt"].is_string()); assert!(response["updatedAt"].is_string()); - let key = response["key"].as_str().unwrap(); - let created_at = response["createdAt"].as_str().unwrap(); - let updated_at = response["updatedAt"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); let content = json!({ "expiresAt": "2055-11-13T00:00:00Z" }); thread::sleep(time::Duration::new(1, 0)); - let (response, code) = server.patch_api_key(&key, content).await; - assert!(response["key"].is_string()); - assert!(response["expiresAt"].is_string()); - assert!(response["createdAt"].is_string()); - assert_ne!(response["updatedAt"].as_str().unwrap(), updated_at); - assert_eq!(response["createdAt"].as_str().unwrap(), created_at); + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(400, code, "{:?}", &response); - let expected = json!({ - "description": "Indexing API key", - "indexes": ["products"], - "actions": [ - "search", - "documents.add", - "documents.get", - "documents.delete", - "indexes.create", - "indexes.get", - "indexes.update", - "indexes.delete", - "stats.get", - "dumps.create", - "dumps.get" - ], - "expiresAt": "2055-11-13T00:00:00Z" + let expected = json!({"message": "The `expiresAt` field cannot be modified for the given resource.", + "code": "immutable_field", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_field" }); assert_json_include!(actual: response, expected: expected); - assert_eq!(code, 200); } #[actix_rt::test] @@ -1126,6 +1284,7 @@ async fn error_patch_api_key_no_header() { json!({}), ) .await; + assert_eq!(401, code, "{:?}", &response); let expected_response = json!({ "message": "The Authorization header is missing. It must use the bearer authorization method.", @@ -1135,7 +1294,6 @@ async fn error_patch_api_key_no_header() { }); assert_eq!(response, expected_response); - assert_eq!(code, 401); } #[actix_rt::test] @@ -1149,6 +1307,7 @@ async fn error_patch_api_key_bad_key() { json!({}), ) .await; + assert_eq!(403, code, "{:?}", &response); let expected_response = json!({ "message": "The provided API key is invalid.", @@ -1158,7 +1317,6 @@ async fn error_patch_api_key_bad_key() { }); assert_eq!(response, expected_response); - assert_eq!(code, 403); } #[actix_rt::test] @@ -1172,6 +1330,7 @@ async fn error_patch_api_key_not_found() { json!({}), ) .await; + assert_eq!(404, code, "{:?}", &response); let expected_response = json!({ "message": "API key `d0552b41d0552b41536279a0ad88bd595327b96f01176a60c2243e906c52ac02375f9bc4` not found.", @@ -1181,7 +1340,6 @@ async fn error_patch_api_key_not_found() { }); assert_eq!(response, expected_response); - assert_eq!(code, 404); } #[actix_rt::test] @@ -1200,17 +1358,18 @@ async fn error_patch_api_key_indexes_invalid_parameters() { let (response, code) = server.add_api_key(content).await; // must pass if add_valid_api_key test passes. - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); - let key = response["key"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); // invalid description let content = json!({ "description": 13 }); - let (response, code) = server.patch_api_key(&key, content).await; + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ "message": "`description` field value `13` is invalid. It should be a string or specified as a null value.", @@ -1220,56 +1379,23 @@ async fn error_patch_api_key_indexes_invalid_parameters() { }); assert_eq!(response, expected_response); - assert_eq!(code, 400); - // invalid indexes + // invalid name let content = json!({ - "indexes": 13 + "name": 13 }); - let (response, code) = server.patch_api_key(&key, content).await; + let (response, code) = server.patch_api_key(&uid, content).await; + assert_eq!(400, code, "{:?}", &response); let expected_response = json!({ - "message": "`indexes` field value `13` is invalid. It should be an array of string representing index names.", - "code": "invalid_api_key_indexes", + "message": "`name` field value `13` is invalid. It should be a string or specified as a null value.", + "code": "invalid_api_key_name", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_api_key_indexes" + "link": "https://docs.meilisearch.com/errors#invalid_api_key_name" }); assert_eq!(response, expected_response); - assert_eq!(code, 400); - - // invalid actions - let content = json!({ - "actions": 13 - }); - let (response, code) = server.patch_api_key(&key, content).await; - - let expected_response = json!({ - "message": "`actions` field value `13` is invalid. It should be an array of string representing action names.", - "code": "invalid_api_key_actions", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" - }); - - assert_eq!(response, expected_response); - assert_eq!(code, 400); - - // invalid expiresAt - let content = json!({ - "expiresAt": 13 - }); - let (response, code) = server.patch_api_key(&key, content).await; - - let expected_response = json!({ - "message": "`expiresAt` field value `13` is invalid. It should follow the RFC 3339 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'.", - "code": "invalid_api_key_expires_at", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_api_key_expires_at" - }); - - assert_eq!(response, expected_response); - assert_eq!(code, 400); } #[actix_rt::test] @@ -1286,23 +1412,23 @@ async fn error_access_api_key_routes_no_master_key_set() { let (response, code) = server.add_api_key(json!({})).await; + assert_eq!(expected_code, code, "{:?}", &response); assert_eq!(response, expected_response); - assert_eq!(code, expected_code); let (response, code) = server.patch_api_key("content", json!({})).await; + assert_eq!(expected_code, code, "{:?}", &response); assert_eq!(response, expected_response); - assert_eq!(code, expected_code); let (response, code) = server.get_api_key("content").await; + assert_eq!(expected_code, code, "{:?}", &response); assert_eq!(response, expected_response); - assert_eq!(code, expected_code); let (response, code) = server.list_api_keys().await; + assert_eq!(expected_code, code, "{:?}", &response); assert_eq!(response, expected_response); - assert_eq!(code, expected_code); server.use_api_key("MASTER_KEY"); @@ -1315,21 +1441,21 @@ async fn error_access_api_key_routes_no_master_key_set() { let (response, code) = server.add_api_key(json!({})).await; + assert_eq!(expected_code, code, "{:?}", &response); assert_eq!(response, expected_response); - assert_eq!(code, expected_code); let (response, code) = server.patch_api_key("content", json!({})).await; + assert_eq!(expected_code, code, "{:?}", &response); assert_eq!(response, expected_response); - assert_eq!(code, expected_code); let (response, code) = server.get_api_key("content").await; + assert_eq!(expected_code, code, "{:?}", &response); assert_eq!(response, expected_response); - assert_eq!(code, expected_code); let (response, code) = server.list_api_keys().await; + assert_eq!(expected_code, code, "{:?}", &response); assert_eq!(response, expected_response); - assert_eq!(code, expected_code); } diff --git a/meilisearch-http/tests/auth/authorization.rs b/meilisearch-http/tests/auth/authorization.rs index 30df2dd2d..e5826a675 100644 --- a/meilisearch-http/tests/auth/authorization.rs +++ b/meilisearch-http/tests/auth/authorization.rs @@ -16,9 +16,9 @@ pub static AUTHORIZATIONS: Lazy hashset!{"documents.get", "*"}, ("DELETE", "/indexes/products/documents/0") => hashset!{"documents.delete", "*"}, ("GET", "/tasks") => hashset!{"tasks.get", "*"}, - ("GET", "/indexes/products/tasks") => hashset!{"tasks.get", "*"}, - ("GET", "/indexes/products/tasks/0") => hashset!{"tasks.get", "*"}, - ("PUT", "/indexes/products/") => hashset!{"indexes.update", "*"}, + ("GET", "/tasks?indexUid=products") => hashset!{"tasks.get", "*"}, + ("GET", "/tasks/0") => hashset!{"tasks.get", "*"}, + ("PATCH", "/indexes/products/") => hashset!{"indexes.update", "*"}, ("GET", "/indexes/products/") => hashset!{"indexes.get", "*"}, ("DELETE", "/indexes/products/") => hashset!{"indexes.delete", "*"}, ("POST", "/indexes") => hashset!{"indexes.create", "*"}, @@ -33,20 +33,25 @@ pub static AUTHORIZATIONS: Lazy hashset!{"settings.get", "*"}, ("GET", "/indexes/products/settings/synonyms") => hashset!{"settings.get", "*"}, ("DELETE", "/indexes/products/settings") => hashset!{"settings.update", "*"}, - ("POST", "/indexes/products/settings") => hashset!{"settings.update", "*"}, - ("POST", "/indexes/products/settings/displayed-attributes") => hashset!{"settings.update", "*"}, - ("POST", "/indexes/products/settings/distinct-attribute") => hashset!{"settings.update", "*"}, - ("POST", "/indexes/products/settings/filterable-attributes") => hashset!{"settings.update", "*"}, - ("POST", "/indexes/products/settings/ranking-rules") => hashset!{"settings.update", "*"}, - ("POST", "/indexes/products/settings/searchable-attributes") => hashset!{"settings.update", "*"}, - ("POST", "/indexes/products/settings/sortable-attributes") => hashset!{"settings.update", "*"}, - ("POST", "/indexes/products/settings/stop-words") => hashset!{"settings.update", "*"}, - ("POST", "/indexes/products/settings/synonyms") => hashset!{"settings.update", "*"}, + ("PATCH", "/indexes/products/settings") => hashset!{"settings.update", "*"}, + ("PATCH", "/indexes/products/settings/typo-tolerance") => hashset!{"settings.update", "*"}, + ("PUT", "/indexes/products/settings/displayed-attributes") => hashset!{"settings.update", "*"}, + ("PUT", "/indexes/products/settings/distinct-attribute") => hashset!{"settings.update", "*"}, + ("PUT", "/indexes/products/settings/filterable-attributes") => hashset!{"settings.update", "*"}, + ("PUT", "/indexes/products/settings/ranking-rules") => hashset!{"settings.update", "*"}, + ("PUT", "/indexes/products/settings/searchable-attributes") => hashset!{"settings.update", "*"}, + ("PUT", "/indexes/products/settings/sortable-attributes") => hashset!{"settings.update", "*"}, + ("PUT", "/indexes/products/settings/stop-words") => hashset!{"settings.update", "*"}, + ("PUT", "/indexes/products/settings/synonyms") => hashset!{"settings.update", "*"}, ("GET", "/indexes/products/stats") => hashset!{"stats.get", "*"}, ("GET", "/stats") => hashset!{"stats.get", "*"}, ("POST", "/dumps") => hashset!{"dumps.create", "*"}, - ("GET", "/dumps/0/status") => hashset!{"dumps.get", "*"}, ("GET", "/version") => hashset!{"version", "*"}, + ("PATCH", "/keys/mykey/") => hashset!{"keys.update", "*"}, + ("GET", "/keys/mykey/") => hashset!{"keys.get", "*"}, + ("DELETE", "/keys/mykey/") => hashset!{"keys.delete", "*"}, + ("POST", "/keys") => hashset!{"keys.create", "*"}, + ("GET", "/keys") => hashset!{"keys.get", "*"}, } }); @@ -81,7 +86,7 @@ async fn error_access_expired_key() { }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); let key = response["key"].as_str().unwrap(); @@ -93,8 +98,14 @@ async fn error_access_expired_key() { for (method, route) in AUTHORIZATIONS.keys() { let (response, code) = server.dummy_request(method, route).await; - assert_eq!(response, INVALID_RESPONSE.clone()); - assert_eq!(code, 403); + assert_eq!( + response, + INVALID_RESPONSE.clone(), + "on route: {:?} - {:?}", + method, + route + ); + assert_eq!(403, code, "{:?}", &response); } } @@ -111,7 +122,7 @@ async fn error_access_unauthorized_index() { }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); let key = response["key"].as_str().unwrap(); @@ -124,8 +135,14 @@ async fn error_access_unauthorized_index() { { let (response, code) = server.dummy_request(method, route).await; - assert_eq!(response, INVALID_RESPONSE.clone()); - assert_eq!(code, 403); + assert_eq!( + response, + INVALID_RESPONSE.clone(), + "on route: {:?} - {:?}", + method, + route + ); + assert_eq!(403, code, "{:?}", &response); } } @@ -133,36 +150,54 @@ async fn error_access_unauthorized_index() { #[cfg_attr(target_os = "windows", ignore)] async fn error_access_unauthorized_action() { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); - - let content = json!({ - "indexes": ["products"], - "actions": [], - "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), - }); - - let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); - assert!(response["key"].is_string()); - - let key = response["key"].as_str().unwrap(); - server.use_api_key(&key); for ((method, route), action) in AUTHORIZATIONS.iter() { + // create a new API key letting only the needed action. server.use_api_key("MASTER_KEY"); - // Patch API key letting all rights but the needed one. let content = json!({ + "indexes": ["products"], "actions": ALL_ACTIONS.difference(action).collect::>(), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); - let (_, code) = server.patch_api_key(&key, content).await; - assert_eq!(code, 200); + let (response, code) = server.add_api_key(content).await; + assert_eq!(201, code, "{:?}", &response); + assert!(response["key"].is_string()); + + let key = response["key"].as_str().unwrap(); server.use_api_key(&key); let (response, code) = server.dummy_request(method, route).await; - assert_eq!(response, INVALID_RESPONSE.clone()); - assert_eq!(code, 403); + assert_eq!( + response, + INVALID_RESPONSE.clone(), + "on route: {:?} - {:?}", + method, + route + ); + assert_eq!(403, code, "{:?}", &response); + } +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn access_authorized_master_key() { + let mut server = Server::new_auth().await; + server.use_api_key("MASTER_KEY"); + + // master key must have access to all routes. + for ((method, route), _) in AUTHORIZATIONS.iter() { + let (response, code) = server.dummy_request(method, route).await; + + assert_ne!( + response, + INVALID_RESPONSE.clone(), + "on route: {:?} - {:?}", + method, + route + ); + assert_ne!(code, 403); } } @@ -170,36 +205,34 @@ async fn error_access_unauthorized_action() { #[cfg_attr(target_os = "windows", ignore)] async fn access_authorized_restricted_index() { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); - - let content = json!({ - "indexes": ["products"], - "actions": [], - "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), - }); - - let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); - assert!(response["key"].is_string()); - - let key = response["key"].as_str().unwrap(); - server.use_api_key(&key); - for ((method, route), actions) in AUTHORIZATIONS.iter() { for action in actions { - // Patch API key letting only the needed action. + // create a new API key letting only the needed action. + server.use_api_key("MASTER_KEY"); + let content = json!({ + "indexes": ["products"], "actions": [action], + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); - server.use_api_key("MASTER_KEY"); - let (_, code) = server.patch_api_key(&key, content).await; - assert_eq!(code, 200); + let (response, code) = server.add_api_key(content).await; + assert_eq!(201, code, "{:?}", &response); + assert!(response["key"].is_string()); + let key = response["key"].as_str().unwrap(); server.use_api_key(&key); + let (response, code) = server.dummy_request(method, route).await; - assert_ne!(response, INVALID_RESPONSE.clone()); + assert_ne!( + response, + INVALID_RESPONSE.clone(), + "on route: {:?} - {:?} with action: {:?}", + method, + route, + action + ); assert_ne!(code, 403); } } @@ -209,36 +242,35 @@ async fn access_authorized_restricted_index() { #[cfg_attr(target_os = "windows", ignore)] async fn access_authorized_no_index_restriction() { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); - - let content = json!({ - "indexes": ["*"], - "actions": [], - "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), - }); - - let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); - assert!(response["key"].is_string()); - - let key = response["key"].as_str().unwrap(); - server.use_api_key(&key); for ((method, route), actions) in AUTHORIZATIONS.iter() { for action in actions { + // create a new API key letting only the needed action. server.use_api_key("MASTER_KEY"); - // Patch API key letting only the needed action. let content = json!({ + "indexes": ["*"], "actions": [action], + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); - let (_, code) = server.patch_api_key(&key, content).await; - assert_eq!(code, 200); + let (response, code) = server.add_api_key(content).await; + assert_eq!(201, code, "{:?}", &response); + assert!(response["key"].is_string()); + + let key = response["key"].as_str().unwrap(); server.use_api_key(&key); + let (response, code) = server.dummy_request(method, route).await; - assert_ne!(response, INVALID_RESPONSE.clone()); + assert_ne!( + response, + INVALID_RESPONSE.clone(), + "on route: {:?} - {:?} with action: {:?}", + method, + route, + action + ); assert_ne!(code, 403); } } @@ -248,16 +280,16 @@ async fn access_authorized_no_index_restriction() { #[cfg_attr(target_os = "windows", ignore)] async fn access_authorized_stats_restricted_index() { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); + server.use_admin_key("MASTER_KEY").await; // create index `test` let index = server.index("test"); - let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("id")).await; + assert_eq!(202, code, "{:?}", &response); // create index `products` let index = server.index("products"); - let (_, code) = index.create(Some("product_id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("product_id")).await; + assert_eq!(202, code, "{:?}", &response); index.wait_task(0).await; // create key with access on `products` index only. @@ -267,7 +299,7 @@ async fn access_authorized_stats_restricted_index() { "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); // use created key. @@ -275,7 +307,7 @@ async fn access_authorized_stats_restricted_index() { server.use_api_key(&key); let (response, code) = server.stats().await; - assert_eq!(code, 200); + assert_eq!(200, code, "{:?}", &response); // key should have access on `products` index. assert!(response["indexes"].get("products").is_some()); @@ -288,16 +320,16 @@ async fn access_authorized_stats_restricted_index() { #[cfg_attr(target_os = "windows", ignore)] async fn access_authorized_stats_no_index_restriction() { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); + server.use_admin_key("MASTER_KEY").await; // create index `test` let index = server.index("test"); - let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("id")).await; + assert_eq!(202, code, "{:?}", &response); // create index `products` let index = server.index("products"); - let (_, code) = index.create(Some("product_id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("product_id")).await; + assert_eq!(202, code, "{:?}", &response); index.wait_task(0).await; // create key with access on all indexes. @@ -307,7 +339,7 @@ async fn access_authorized_stats_no_index_restriction() { "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); // use created key. @@ -315,7 +347,7 @@ async fn access_authorized_stats_no_index_restriction() { server.use_api_key(&key); let (response, code) = server.stats().await; - assert_eq!(code, 200); + assert_eq!(200, code, "{:?}", &response); // key should have access on `products` index. assert!(response["indexes"].get("products").is_some()); @@ -328,16 +360,16 @@ async fn access_authorized_stats_no_index_restriction() { #[cfg_attr(target_os = "windows", ignore)] async fn list_authorized_indexes_restricted_index() { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); + server.use_admin_key("MASTER_KEY").await; // create index `test` let index = server.index("test"); - let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("id")).await; + assert_eq!(202, code, "{:?}", &response); // create index `products` let index = server.index("products"); - let (_, code) = index.create(Some("product_id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("product_id")).await; + assert_eq!(202, code, "{:?}", &response); index.wait_task(0).await; // create key with access on `products` index only. @@ -347,17 +379,17 @@ async fn list_authorized_indexes_restricted_index() { "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); // use created key. let key = response["key"].as_str().unwrap(); server.use_api_key(&key); - let (response, code) = server.list_indexes().await; - assert_eq!(code, 200); + let (response, code) = server.list_indexes(None, None).await; + assert_eq!(200, code, "{:?}", &response); - let response = response.as_array().unwrap(); + let response = response["results"].as_array().unwrap(); // key should have access on `products` index. assert!(response.iter().any(|index| index["uid"] == "products")); @@ -369,16 +401,16 @@ async fn list_authorized_indexes_restricted_index() { #[cfg_attr(target_os = "windows", ignore)] async fn list_authorized_indexes_no_index_restriction() { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); + server.use_admin_key("MASTER_KEY").await; // create index `test` let index = server.index("test"); - let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("id")).await; + assert_eq!(202, code, "{:?}", &response); // create index `products` let index = server.index("products"); - let (_, code) = index.create(Some("product_id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("product_id")).await; + assert_eq!(202, code, "{:?}", &response); index.wait_task(0).await; // create key with access on all indexes. @@ -388,17 +420,17 @@ async fn list_authorized_indexes_no_index_restriction() { "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); // use created key. let key = response["key"].as_str().unwrap(); server.use_api_key(&key); - let (response, code) = server.list_indexes().await; - assert_eq!(code, 200); + let (response, code) = server.list_indexes(None, None).await; + assert_eq!(200, code, "{:?}", &response); - let response = response.as_array().unwrap(); + let response = response["results"].as_array().unwrap(); // key should have access on `products` index. assert!(response.iter().any(|index| index["uid"] == "products")); @@ -409,16 +441,16 @@ async fn list_authorized_indexes_no_index_restriction() { #[actix_rt::test] async fn list_authorized_tasks_restricted_index() { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); + server.use_admin_key("MASTER_KEY").await; // create index `test` let index = server.index("test"); - let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("id")).await; + assert_eq!(202, code, "{:?}", &response); // create index `products` let index = server.index("products"); - let (_, code) = index.create(Some("product_id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("product_id")).await; + assert_eq!(202, code, "{:?}", &response); index.wait_task(0).await; // create key with access on `products` index only. @@ -428,7 +460,7 @@ async fn list_authorized_tasks_restricted_index() { "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); // use created key. @@ -436,7 +468,7 @@ async fn list_authorized_tasks_restricted_index() { server.use_api_key(&key); let (response, code) = server.service.get("/tasks").await; - assert_eq!(code, 200); + assert_eq!(200, code, "{:?}", &response); println!("{}", response); let response = response["results"].as_array().unwrap(); // key should have access on `products` index. @@ -449,16 +481,16 @@ async fn list_authorized_tasks_restricted_index() { #[actix_rt::test] async fn list_authorized_tasks_no_index_restriction() { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); + server.use_admin_key("MASTER_KEY").await; // create index `test` let index = server.index("test"); - let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("id")).await; + assert_eq!(202, code, "{:?}", &response); // create index `products` let index = server.index("products"); - let (_, code) = index.create(Some("product_id")).await; - assert_eq!(code, 202); + let (response, code) = index.create(Some("product_id")).await; + assert_eq!(202, code, "{:?}", &response); index.wait_task(0).await; // create key with access on all indexes. @@ -468,7 +500,7 @@ async fn list_authorized_tasks_no_index_restriction() { "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); // use created key. @@ -476,7 +508,7 @@ async fn list_authorized_tasks_no_index_restriction() { server.use_api_key(&key); let (response, code) = server.service.get("/tasks").await; - assert_eq!(code, 200); + assert_eq!(200, code, "{:?}", &response); let response = response["results"].as_array().unwrap(); // key should have access on `products` index. @@ -499,7 +531,7 @@ async fn error_creating_index_without_action() { "expiresAt": "2050-11-13T00:00:00Z" }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); // use created key. @@ -523,8 +555,8 @@ async fn error_creating_index_without_action() { ]); let (response, code) = index.add_documents(documents, None).await; - assert_eq!(code, 202, "{:?}", response); - let task_id = response["uid"].as_u64().unwrap(); + assert_eq!(202, code, "{:?}", &response); + let task_id = response["taskUid"].as_u64().unwrap(); let response = index.wait_task(task_id).await; assert_eq!(response["status"], "failed"); @@ -534,8 +566,8 @@ async fn error_creating_index_without_action() { let settings = json!({ "distinctAttribute": "test"}); let (response, code) = index.update_settings(settings).await; - assert_eq!(code, 202); - let task_id = response["uid"].as_u64().unwrap(); + assert_eq!(202, code, "{:?}", &response); + let task_id = response["taskUid"].as_u64().unwrap(); let response = index.wait_task(task_id).await; @@ -544,8 +576,8 @@ async fn error_creating_index_without_action() { // try to create a index via add specialized settings route let (response, code) = index.update_distinct_attribute(json!("test")).await; - assert_eq!(code, 202); - let task_id = response["uid"].as_u64().unwrap(); + assert_eq!(202, code, "{:?}", &response); + let task_id = response["taskUid"].as_u64().unwrap(); let response = index.wait_task(task_id).await; @@ -566,7 +598,7 @@ async fn lazy_create_index() { }); let (response, code) = server.add_api_key(content).await; - assert_eq!(code, 201); + assert_eq!(201, code, "{:?}", &response); assert!(response["key"].is_string()); // use created key. @@ -583,13 +615,13 @@ async fn lazy_create_index() { ]); let (response, code) = index.add_documents(documents, None).await; - assert_eq!(code, 202, "{:?}", response); - let task_id = response["uid"].as_u64().unwrap(); + assert_eq!(202, code, "{:?}", &response); + let task_id = response["taskUid"].as_u64().unwrap(); index.wait_task(task_id).await; let (response, code) = index.get_task(task_id).await; - assert_eq!(code, 200); + assert_eq!(200, code, "{:?}", &response); assert_eq!(response["status"], "succeeded"); // try to create a index via add settings route @@ -597,24 +629,24 @@ async fn lazy_create_index() { let settings = json!({ "distinctAttribute": "test"}); let (response, code) = index.update_settings(settings).await; - assert_eq!(code, 202); - let task_id = response["uid"].as_u64().unwrap(); + assert_eq!(202, code, "{:?}", &response); + let task_id = response["taskUid"].as_u64().unwrap(); index.wait_task(task_id).await; let (response, code) = index.get_task(task_id).await; - assert_eq!(code, 200); + assert_eq!(200, code, "{:?}", &response); assert_eq!(response["status"], "succeeded"); // try to create a index via add specialized settings route let index = server.index("test2"); let (response, code) = index.update_distinct_attribute(json!("test")).await; - assert_eq!(code, 202); - let task_id = response["uid"].as_u64().unwrap(); + assert_eq!(202, code, "{:?}", &response); + let task_id = response["taskUid"].as_u64().unwrap(); index.wait_task(task_id).await; let (response, code) = index.get_task(task_id).await; - assert_eq!(code, 200); + assert_eq!(200, code, "{:?}", &response); assert_eq!(response["status"], "succeeded"); } diff --git a/meilisearch-http/tests/auth/mod.rs b/meilisearch-http/tests/auth/mod.rs index ef47f4a6a..03c24dd6d 100644 --- a/meilisearch-http/tests/auth/mod.rs +++ b/meilisearch-http/tests/auth/mod.rs @@ -13,6 +13,15 @@ impl Server { self.service.api_key = Some(api_key.as_ref().to_string()); } + /// Fetch and use the default admin key for nexts http requests. + pub async fn use_admin_key(&mut self, master_key: impl AsRef) { + self.use_api_key(master_key); + let (response, code) = self.list_api_keys().await; + assert_eq!(200, code, "{:?}", response); + let admin_key = &response["results"][1]["key"]; + self.use_api_key(admin_key.as_str().unwrap()); + } + pub async fn add_api_key(&self, content: Value) -> (Value, StatusCode) { let url = "/keys"; self.service.post(url, content).await diff --git a/meilisearch-http/tests/auth/tenant_token.rs b/meilisearch-http/tests/auth/tenant_token.rs index bb9224590..d82e170aa 100644 --- a/meilisearch-http/tests/auth/tenant_token.rs +++ b/meilisearch-http/tests/auth/tenant_token.rs @@ -8,11 +8,15 @@ use time::{Duration, OffsetDateTime}; use super::authorization::{ALL_ACTIONS, AUTHORIZATIONS}; -fn generate_tenant_token(parent_key: impl AsRef, mut body: HashMap<&str, Value>) -> String { +fn generate_tenant_token( + parent_uid: impl AsRef, + parent_key: impl AsRef, + mut body: HashMap<&str, Value>, +) -> String { use jsonwebtoken::{encode, EncodingKey, Header}; - let key_id = &parent_key.as_ref()[..8]; - body.insert("apiKeyPrefix", json!(key_id)); + let parent_uid = parent_uid.as_ref(); + body.insert("apiKeyUid", json!(parent_uid)); encode( &Header::default(), &body, @@ -114,7 +118,7 @@ static REFUSED_KEYS: Lazy> = Lazy::new(|| { macro_rules! compute_autorized_search { ($tenant_tokens:expr, $filter:expr, $expected_count:expr) => { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); + server.use_admin_key("MASTER_KEY").await; let index = server.index("sales"); let documents = DOCUMENTS.clone(); index.add_documents(documents, None).await; @@ -130,9 +134,10 @@ macro_rules! compute_autorized_search { let (response, code) = server.add_api_key(key_content.clone()).await; assert_eq!(code, 201); let key = response["key"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); for tenant_token in $tenant_tokens.iter() { - let web_token = generate_tenant_token(&key, tenant_token.clone()); + let web_token = generate_tenant_token(&uid, &key, tenant_token.clone()); server.use_api_key(&web_token); let index = server.index("sales"); index @@ -160,7 +165,7 @@ macro_rules! compute_autorized_search { macro_rules! compute_forbidden_search { ($tenant_tokens:expr, $parent_keys:expr) => { let mut server = Server::new_auth().await; - server.use_api_key("MASTER_KEY"); + server.use_admin_key("MASTER_KEY").await; let index = server.index("sales"); let documents = DOCUMENTS.clone(); index.add_documents(documents, None).await; @@ -172,9 +177,10 @@ macro_rules! compute_forbidden_search { let (response, code) = server.add_api_key(key_content.clone()).await; assert_eq!(code, 201, "{:?}", response); let key = response["key"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); for tenant_token in $tenant_tokens.iter() { - let web_token = generate_tenant_token(&key, tenant_token.clone()); + let web_token = generate_tenant_token(&uid, &key, tenant_token.clone()); server.use_api_key(&web_token); let index = server.index("sales"); index @@ -461,12 +467,13 @@ async fn error_access_forbidden_routes() { assert!(response["key"].is_string()); let key = response["key"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); let tenant_token = hashmap! { "searchRules" => json!(["*"]), "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }; - let web_token = generate_tenant_token(&key, tenant_token); + let web_token = generate_tenant_token(&uid, &key, tenant_token); server.use_api_key(&web_token); for ((method, route), actions) in AUTHORIZATIONS.iter() { @@ -496,12 +503,13 @@ async fn error_access_expired_parent_key() { assert!(response["key"].is_string()); let key = response["key"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); let tenant_token = hashmap! { "searchRules" => json!(["*"]), "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }; - let web_token = generate_tenant_token(&key, tenant_token); + let web_token = generate_tenant_token(&uid, &key, tenant_token); server.use_api_key(&web_token); // test search request while parent_key is not expired @@ -538,12 +546,13 @@ async fn error_access_modified_token() { assert!(response["key"].is_string()); let key = response["key"].as_str().unwrap(); + let uid = response["uid"].as_str().unwrap(); let tenant_token = hashmap! { "searchRules" => json!(["products"]), "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }; - let web_token = generate_tenant_token(&key, tenant_token); + let web_token = generate_tenant_token(&uid, &key, tenant_token); server.use_api_key(&web_token); // test search request while web_token is valid @@ -558,7 +567,7 @@ async fn error_access_modified_token() { "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }; - let alt = generate_tenant_token(&key, tenant_token); + let alt = generate_tenant_token(&uid, &key, tenant_token); let altered_token = [ web_token.split('.').next().unwrap(), alt.split('.').nth(1).unwrap(), diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 6c44ea369..90d138ced 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -1,32 +1,16 @@ use std::{ + fmt::Write, panic::{catch_unwind, resume_unwind, UnwindSafe}, time::Duration, }; use actix_web::http::StatusCode; -use paste::paste; use serde_json::{json, Value}; use tokio::time::sleep; use urlencoding::encode; use super::service::Service; -macro_rules! make_settings_test_routes { - ($($name:ident),+) => { - $(paste! { - pub async fn [](&self, value: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/{}", encode(self.uid.as_ref()).to_string(), stringify!($name).replace("_", "-")); - self.service.post(url, value).await - } - - pub async fn [](&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/{}", encode(self.uid.as_ref()).to_string(), stringify!($name).replace("_", "-")); - self.service.get(url).await - } - })* - }; -} - pub struct Index<'a> { pub uid: String, pub service: &'a Service, @@ -46,7 +30,7 @@ impl Index<'_> { .post_str(url, include_str!("../assets/test_set.json")) .await; assert_eq!(code, 202); - let update_id = response["uid"].as_i64().unwrap(); + let update_id = response["taskUid"].as_i64().unwrap(); self.wait_task(update_id as u64).await; update_id as u64 } @@ -65,7 +49,7 @@ impl Index<'_> { }); let url = format!("/indexes/{}", encode(self.uid.as_ref())); - self.service.put(url, body).await + self.service.patch(url, body).await } pub async fn delete(&self) -> (Value, StatusCode) { @@ -106,55 +90,67 @@ impl Index<'_> { } pub async fn wait_task(&self, update_id: u64) -> Value { - // try 10 times to get status, or panic to not wait forever + // try several times to get status, or panic to not wait forever let url = format!("/tasks/{}", update_id); - for _ in 0..10 { + for _ in 0..100 { let (response, status_code) = self.service.get(&url).await; - assert_eq!(status_code, 200, "response: {}", response); + assert_eq!(200, status_code, "response: {}", response); if response["status"] == "succeeded" || response["status"] == "failed" { return response; } - sleep(Duration::from_secs(1)).await; + // wait 0.5 second. + sleep(Duration::from_millis(500)).await; } panic!("Timeout waiting for update id"); } pub async fn get_task(&self, update_id: u64) -> (Value, StatusCode) { - let url = format!("/indexes/{}/tasks/{}", self.uid, update_id); + let url = format!("/tasks/{}", update_id); self.service.get(url).await } pub async fn list_tasks(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/tasks", self.uid); + let url = format!("/tasks?indexUid={}", self.uid); + self.service.get(url).await + } + + pub async fn filtered_tasks(&self, type_: &[&str], status: &[&str]) -> (Value, StatusCode) { + let mut url = format!("/tasks?indexUid={}", self.uid); + if !type_.is_empty() { + let _ = write!(url, "&type={}", type_.join(",")); + } + if !status.is_empty() { + let _ = write!(url, "&status={}", status.join(",")); + } self.service.get(url).await } pub async fn get_document( &self, id: u64, - _options: Option, + options: Option, ) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); + let mut url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); + if let Some(fields) = options.and_then(|o| o.fields) { + let _ = write!(url, "?fields={}", fields.join(",")); + } self.service.get(url).await } pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) { let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref())); if let Some(limit) = options.limit { - url.push_str(&format!("limit={}&", limit)); + let _ = write!(url, "limit={}&", limit); } if let Some(offset) = options.offset { - url.push_str(&format!("offset={}&", offset)); + let _ = write!(url, "offset={}&", offset); } if let Some(attributes_to_retrieve) = options.attributes_to_retrieve { - url.push_str(&format!( - "attributesToRetrieve={}&", - attributes_to_retrieve.join(",") - )); + let _ = write!(url, "fields={}&", attributes_to_retrieve.join(",")); } self.service.get(url).await @@ -187,7 +183,7 @@ impl Index<'_> { pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) { let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); - self.service.post(url, settings).await + self.service.patch(url, settings).await } pub async fn delete_settings(&self) -> (Value, StatusCode) { @@ -226,15 +222,33 @@ impl Index<'_> { } pub async fn search_get(&self, query: Value) -> (Value, StatusCode) { - let params = serde_url_params::to_string(&query).unwrap(); + let params = yaup::to_string(&query).unwrap(); let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params); self.service.get(url).await } - make_settings_test_routes!(distinct_attribute); + pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) { + let url = format!( + "/indexes/{}/settings/{}", + encode(self.uid.as_ref()), + "distinct-attribute" + ); + self.service.put(url, value).await + } + + pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) { + let url = format!( + "/indexes/{}/settings/{}", + encode(self.uid.as_ref()), + "distinct-attribute" + ); + self.service.get(url).await + } } -pub struct GetDocumentOptions; +pub struct GetDocumentOptions { + pub fields: Option>, +} #[derive(Debug, Default)] pub struct GetAllDocumentsOptions { diff --git a/meilisearch-http/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs index e734b3621..b076b0ea5 100644 --- a/meilisearch-http/tests/common/mod.rs +++ b/meilisearch-http/tests/common/mod.rs @@ -3,7 +3,7 @@ pub mod server; pub mod service; pub use index::{GetAllDocumentsOptions, GetDocumentOptions}; -pub use server::Server; +pub use server::{default_settings, Server}; /// Performs a search test on both post and get routes #[macro_export] diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index b439ec52e..146690766 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -52,16 +52,13 @@ impl Server { } } - pub async fn new_auth() -> Self { - let dir = TempDir::new().unwrap(); - + pub async fn new_auth_with_options(mut options: Opt, dir: TempDir) -> Self { if cfg!(windows) { std::env::set_var("TMP", TEST_TEMP_DIR.path()); } else { std::env::set_var("TMPDIR", TEST_TEMP_DIR.path()); } - let mut options = default_settings(dir.path()); options.master_key = Some("MASTER_KEY".to_string()); let meilisearch = setup_meilisearch(&options).unwrap(); @@ -79,9 +76,15 @@ impl Server { } } - pub async fn new_with_options(options: Opt) -> Self { - let meilisearch = setup_meilisearch(&options).unwrap(); - let auth = AuthController::new(&options.db_path, &options.master_key).unwrap(); + pub async fn new_auth() -> Self { + let dir = TempDir::new().unwrap(); + let options = default_settings(dir.path()); + Self::new_auth_with_options(options, dir).await + } + + pub async fn new_with_options(options: Opt) -> Result { + let meilisearch = setup_meilisearch(&options)?; + let auth = AuthController::new(&options.db_path, &options.master_key)?; let service = Service { meilisearch, auth, @@ -89,10 +92,10 @@ impl Server { api_key: None, }; - Server { + Ok(Server { service, _dir: None, - } + }) } /// Returns a view to an index. There is no guarantee that the index exists. @@ -103,8 +106,27 @@ impl Server { } } - pub async fn list_indexes(&self) -> (Value, StatusCode) { - self.service.get("/indexes").await + pub async fn list_indexes( + &self, + offset: Option, + limit: Option, + ) -> (Value, StatusCode) { + let (offset, limit) = ( + offset.map(|offset| format!("offset={offset}")), + limit.map(|limit| format!("limit={limit}")), + ); + let query_parameter = offset + .as_ref() + .zip(limit.as_ref()) + .map(|(offset, limit)| format!("{offset}&{limit}")) + .or_else(|| offset.xor(limit)); + if let Some(query_parameter) = query_parameter { + self.service + .get(format!("/indexes?{query_parameter}")) + .await + } else { + self.service.get("/indexes").await + } } pub async fn version(&self) -> (Value, StatusCode) { @@ -131,8 +153,8 @@ pub fn default_settings(dir: impl AsRef) -> Opt { env: "development".to_owned(), #[cfg(all(not(debug_assertions), feature = "analytics"))] no_analytics: true, - max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), - max_task_db_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), + max_index_size: Byte::from_unit(100.0, ByteUnit::MiB).unwrap(), + max_task_db_size: Byte::from_unit(1.0, ByteUnit::GiB).unwrap(), http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), snapshot_dir: ".".into(), indexer_options: IndexerOpts { diff --git a/meilisearch-http/tests/content_type.rs b/meilisearch-http/tests/content_type.rs index d6b4cbd78..eace67a08 100644 --- a/meilisearch-http/tests/content_type.rs +++ b/meilisearch-http/tests/content_type.rs @@ -7,23 +7,45 @@ use actix_web::test; use meilisearch_http::{analytics, create_app}; use serde_json::{json, Value}; +enum HttpVerb { + Put, + Patch, + Post, + Get, + Delete, +} + +impl HttpVerb { + fn test_request(&self) -> test::TestRequest { + match self { + HttpVerb::Put => test::TestRequest::put(), + HttpVerb::Patch => test::TestRequest::patch(), + HttpVerb::Post => test::TestRequest::post(), + HttpVerb::Get => test::TestRequest::get(), + HttpVerb::Delete => test::TestRequest::delete(), + } + } +} + #[actix_rt::test] async fn error_json_bad_content_type() { + use HttpVerb::{Patch, Post, Put}; + let routes = [ - // all the POST routes except the dumps that can be created without any body or content-type + // all the routes except the dumps that can be created without any body or content-type // and the search that is not a strict json - "/indexes", - "/indexes/doggo/documents/delete-batch", - "/indexes/doggo/search", - "/indexes/doggo/settings", - "/indexes/doggo/settings/displayed-attributes", - "/indexes/doggo/settings/distinct-attribute", - "/indexes/doggo/settings/filterable-attributes", - "/indexes/doggo/settings/ranking-rules", - "/indexes/doggo/settings/searchable-attributes", - "/indexes/doggo/settings/sortable-attributes", - "/indexes/doggo/settings/stop-words", - "/indexes/doggo/settings/synonyms", + (Post, "/indexes"), + (Post, "/indexes/doggo/documents/delete-batch"), + (Post, "/indexes/doggo/search"), + (Patch, "/indexes/doggo/settings"), + (Put, "/indexes/doggo/settings/displayed-attributes"), + (Put, "/indexes/doggo/settings/distinct-attribute"), + (Put, "/indexes/doggo/settings/filterable-attributes"), + (Put, "/indexes/doggo/settings/ranking-rules"), + (Put, "/indexes/doggo/settings/searchable-attributes"), + (Put, "/indexes/doggo/settings/sortable-attributes"), + (Put, "/indexes/doggo/settings/stop-words"), + (Put, "/indexes/doggo/settings/synonyms"), ]; let bad_content_types = [ "application/csv", @@ -45,10 +67,11 @@ async fn error_json_bad_content_type() { analytics::MockAnalytics::new(&server.service.options).0 )) .await; - for route in routes { + for (verb, route) in routes { // Good content-type, we probably have an error since we didn't send anything in the json // so we only ensure we didn't get a bad media type error. - let req = test::TestRequest::post() + let req = verb + .test_request() .uri(route) .set_payload(document) .insert_header(("content-type", "application/json")) @@ -59,7 +82,8 @@ async fn error_json_bad_content_type() { "calling the route `{}` with a content-type of json isn't supposed to throw a bad media type error", route); // No content-type. - let req = test::TestRequest::post() + let req = verb + .test_request() .uri(route) .set_payload(document) .to_request(); @@ -82,7 +106,8 @@ async fn error_json_bad_content_type() { for bad_content_type in bad_content_types { // Always bad content-type - let req = test::TestRequest::post() + let req = verb + .test_request() .uri(route) .set_payload(document.to_string()) .insert_header(("content-type", bad_content_type)) diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 911cfd312..c3baf0cb0 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -35,7 +35,7 @@ async fn add_documents_test_json_content_types() { let body = test::read_body(res).await; let response: Value = serde_json::from_slice(&body).unwrap_or_default(); assert_eq!(status_code, 202); - assert_eq!(response["uid"], 0); + assert_eq!(response["taskUid"], 0); // put let req = test::TestRequest::put() @@ -48,7 +48,7 @@ async fn add_documents_test_json_content_types() { let body = test::read_body(res).await; let response: Value = serde_json::from_slice(&body).unwrap_or_default(); assert_eq!(status_code, 202); - assert_eq!(response["uid"], 1); + assert_eq!(response["taskUid"], 1); } /// any other content-type is must be refused @@ -599,7 +599,7 @@ async fn add_documents_no_index_creation() { let (response, code) = index.add_documents(documents, None).await; assert_eq!(code, 202); - assert_eq!(response["uid"], 0); + assert_eq!(response["taskUid"], 0); /* * currently we don’t check these field to stay ISO with meilisearch * assert_eq!(response["status"], "pending"); @@ -615,7 +615,7 @@ async fn add_documents_no_index_creation() { assert_eq!(code, 200); assert_eq!(response["status"], "succeeded"); assert_eq!(response["uid"], 0); - assert_eq!(response["type"], "documentAddition"); + assert_eq!(response["type"], "documentAdditionOrUpdate"); assert_eq!(response["details"]["receivedDocuments"], 1); assert_eq!(response["details"]["indexedDocuments"], 1); @@ -638,7 +638,7 @@ async fn error_document_add_create_index_bad_uid() { let (response, code) = index.add_documents(json!([{"id": 1}]), None).await; let expected_response = json!({ - "message": "`883 fj!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "invalid index uid `883 fj!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -655,7 +655,7 @@ async fn error_document_update_create_index_bad_uid() { let (response, code) = index.update_documents(json!([{"id": 1}]), None).await; let expected_response = json!({ - "message": "`883 fj!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "invalid index uid `883 fj!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -685,7 +685,7 @@ async fn document_addition_with_primary_key() { assert_eq!(code, 200); assert_eq!(response["status"], "succeeded"); assert_eq!(response["uid"], 0); - assert_eq!(response["type"], "documentAddition"); + assert_eq!(response["type"], "documentAdditionOrUpdate"); assert_eq!(response["details"]["receivedDocuments"], 1); assert_eq!(response["details"]["indexedDocuments"], 1); @@ -714,7 +714,7 @@ async fn document_update_with_primary_key() { assert_eq!(code, 200); assert_eq!(response["status"], "succeeded"); assert_eq!(response["uid"], 0); - assert_eq!(response["type"], "documentPartial"); + assert_eq!(response["type"], "documentAdditionOrUpdate"); assert_eq!(response["details"]["indexedDocuments"], 1); assert_eq!(response["details"]["receivedDocuments"], 1); @@ -818,7 +818,7 @@ async fn add_larger_dataset() { let (response, code) = index.get_task(update_id).await; assert_eq!(code, 200); assert_eq!(response["status"], "succeeded"); - assert_eq!(response["type"], "documentAddition"); + assert_eq!(response["type"], "documentAdditionOrUpdate"); assert_eq!(response["details"]["indexedDocuments"], 77); assert_eq!(response["details"]["receivedDocuments"], 77); let (response, code) = index @@ -827,8 +827,8 @@ async fn add_larger_dataset() { ..Default::default() }) .await; - assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 77); + assert_eq!(code, 200, "failed with `{}`", response); + assert_eq!(response["results"].as_array().unwrap().len(), 77); } #[actix_rt::test] @@ -840,7 +840,7 @@ async fn update_larger_dataset() { index.wait_task(0).await; let (response, code) = index.get_task(0).await; assert_eq!(code, 200); - assert_eq!(response["type"], "documentPartial"); + assert_eq!(response["type"], "documentAdditionOrUpdate"); assert_eq!(response["details"]["indexedDocuments"], 77); let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -849,7 +849,7 @@ async fn update_larger_dataset() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 77); + assert_eq!(response["results"].as_array().unwrap().len(), 77); } #[actix_rt::test] @@ -868,7 +868,12 @@ async fn error_add_documents_bad_document_id() { let (response, code) = index.get_task(1).await; assert_eq!(code, 200); assert_eq!(response["status"], json!("failed")); - assert_eq!(response["error"]["message"], json!("Document identifier `foo & bar` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_).")); + assert_eq!( + response["error"]["message"], + json!( + r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."# + ) + ); assert_eq!(response["error"]["code"], json!("invalid_document_id")); assert_eq!(response["error"]["type"], json!("invalid_request")); assert_eq!( @@ -891,7 +896,12 @@ async fn error_update_documents_bad_document_id() { index.update_documents(documents, None).await; let response = index.wait_task(1).await; assert_eq!(response["status"], json!("failed")); - assert_eq!(response["error"]["message"], json!("Document identifier `foo & bar` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_).")); + assert_eq!( + response["error"]["message"], + json!( + r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."# + ) + ); assert_eq!(response["error"]["code"], json!("invalid_document_id")); assert_eq!(response["error"]["type"], json!("invalid_request")); assert_eq!( diff --git a/meilisearch-http/tests/documents/delete_documents.rs b/meilisearch-http/tests/documents/delete_documents.rs index 5198b2bfb..8c7ddaa7b 100644 --- a/meilisearch-http/tests/documents/delete_documents.rs +++ b/meilisearch-http/tests/documents/delete_documents.rs @@ -72,7 +72,7 @@ async fn clear_all_documents() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert!(response.as_array().unwrap().is_empty()); + assert!(response["results"].as_array().unwrap().is_empty()); } #[actix_rt::test] @@ -89,7 +89,7 @@ async fn clear_all_documents_empty_index() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert!(response.as_array().unwrap().is_empty()); + assert!(response["results"].as_array().unwrap().is_empty()); } #[actix_rt::test] @@ -125,8 +125,8 @@ async fn delete_batch() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 1); - assert_eq!(response.as_array().unwrap()[0]["id"], 3); + assert_eq!(response["results"].as_array().unwrap().len(), 1); + assert_eq!(response["results"][0]["id"], json!(3)); } #[actix_rt::test] @@ -143,5 +143,5 @@ async fn delete_no_document_batch() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 3); + assert_eq!(response["results"].as_array().unwrap().len(), 3); } diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index 6c93b9c13..c15d3f7fa 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -1,5 +1,4 @@ -use crate::common::GetAllDocumentsOptions; -use crate::common::Server; +use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server}; use serde_json::json; @@ -39,19 +38,51 @@ async fn get_document() { let documents = serde_json::json!([ { "id": 0, - "content": "foobar", + "nested": { "content": "foobar" }, } ]); let (_, code) = index.add_documents(documents, None).await; assert_eq!(code, 202); - index.wait_task(0).await; + index.wait_task(1).await; let (response, code) = index.get_document(0, None).await; assert_eq!(code, 200); assert_eq!( response, - serde_json::json!( { + serde_json::json!({ "id": 0, - "content": "foobar", + "nested": { "content": "foobar" }, + }) + ); + + let (response, code) = index + .get_document( + 0, + Some(GetDocumentOptions { + fields: Some(vec!["id"]), + }), + ) + .await; + assert_eq!(code, 200); + assert_eq!( + response, + serde_json::json!({ + "id": 0, + }) + ); + + let (response, code) = index + .get_document( + 0, + Some(GetDocumentOptions { + fields: Some(vec!["nested.content"]), + }), + ) + .await; + assert_eq!(code, 200); + assert_eq!( + response, + serde_json::json!({ + "nested": { "content": "foobar" }, }) ); } @@ -88,7 +119,7 @@ async fn get_no_document() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert!(response.as_array().unwrap().is_empty()); + assert!(response["results"].as_array().unwrap().is_empty()); } #[actix_rt::test] @@ -101,7 +132,7 @@ async fn get_all_documents_no_options() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - let arr = response.as_array().unwrap(); + let arr = response["results"].as_array().unwrap(); assert_eq!(arr.len(), 20); let first = serde_json::json!({ "id":0, @@ -137,8 +168,11 @@ async fn test_get_all_documents_limit() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 5); - assert_eq!(response.as_array().unwrap()[0]["id"], 0); + assert_eq!(response["results"].as_array().unwrap().len(), 5); + assert_eq!(response["results"][0]["id"], json!(0)); + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(5)); + assert_eq!(response["total"], json!(77)); } #[actix_rt::test] @@ -154,8 +188,11 @@ async fn test_get_all_documents_offset() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!(response.as_array().unwrap()[0]["id"], 5); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + assert_eq!(response["results"][0]["id"], json!(5)); + assert_eq!(response["offset"], json!(5)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); } #[actix_rt::test] @@ -171,20 +208,14 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 1 - ); - assert!(response.as_array().unwrap()[0] - .as_object() - .unwrap() - .get("name") - .is_some()); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 1); + assert!(results["name"] != json!(null)); + } + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -193,15 +224,13 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 0 - ); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 0); + } + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -210,15 +239,13 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 0 - ); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 0); + } + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -227,15 +254,12 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 2 - ); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 2); + assert!(results["name"] != json!(null)); + assert!(results["tags"] != json!(null)); + } let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -244,15 +268,10 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); - assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 16 - ); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 16); + } let (response, code) = index .get_all_documents(GetAllDocumentsOptions { @@ -261,19 +280,99 @@ async fn test_get_all_documents_attributes_to_retrieve() { }) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response["results"].as_array().unwrap().len(), 20); + for results in response["results"].as_array().unwrap() { + assert_eq!(results.as_object().unwrap().keys().count(), 16); + } +} + +#[actix_rt::test] +async fn get_document_s_nested_attributes_to_retrieve() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + let documents = json!([ + { + "id": 0, + "content.truc": "foobar", + }, + { + "id": 1, + "content": { + "truc": "foobar", + "machin": "bidule", + }, + }, + ]); + let (_, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + index.wait_task(1).await; + + let (response, code) = index + .get_document( + 0, + Some(GetDocumentOptions { + fields: Some(vec!["content"]), + }), + ) + .await; + assert_eq!(code, 200); + assert_eq!(response, json!({})); + let (response, code) = index + .get_document( + 1, + Some(GetDocumentOptions { + fields: Some(vec!["content"]), + }), + ) + .await; + assert_eq!(code, 200); assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 16 + response, + json!({ + "content": { + "truc": "foobar", + "machin": "bidule", + }, + }) + ); + + let (response, code) = index + .get_document( + 0, + Some(GetDocumentOptions { + fields: Some(vec!["content.truc"]), + }), + ) + .await; + assert_eq!(code, 200); + assert_eq!( + response, + json!({ + "content.truc": "foobar", + }) + ); + let (response, code) = index + .get_document( + 1, + Some(GetDocumentOptions { + fields: Some(vec!["content.truc"]), + }), + ) + .await; + assert_eq!(code, 200); + assert_eq!( + response, + json!({ + "content": { + "truc": "foobar", + }, + }) ); } #[actix_rt::test] -async fn get_documents_displayed_attributes() { +async fn get_documents_displayed_attributes_is_ignored() { let server = Server::new().await; let index = server.index("test"); index @@ -285,23 +384,19 @@ async fn get_documents_displayed_attributes() { .get_all_documents(GetAllDocumentsOptions::default()) .await; assert_eq!(code, 200); - assert_eq!(response.as_array().unwrap().len(), 20); + assert_eq!(response["results"].as_array().unwrap().len(), 20); assert_eq!( - response.as_array().unwrap()[0] - .as_object() - .unwrap() - .keys() - .count(), - 1 + response["results"][0].as_object().unwrap().keys().count(), + 16 ); - assert!(response.as_array().unwrap()[0] - .as_object() - .unwrap() - .get("gender") - .is_some()); + assert!(response["results"][0]["gender"] != json!(null)); + + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["total"], json!(77)); let (response, code) = index.get_document(0, None).await; assert_eq!(code, 200); - assert_eq!(response.as_object().unwrap().keys().count(), 1); + assert_eq!(response.as_object().unwrap().keys().count(), 16); assert!(response.as_object().unwrap().get("gender").is_some()); } diff --git a/meilisearch-http/tests/dumps.rs b/meilisearch-http/tests/dumps.rs deleted file mode 100644 index 843347bde..000000000 --- a/meilisearch-http/tests/dumps.rs +++ /dev/null @@ -1,22 +0,0 @@ -#![allow(dead_code)] -mod common; - -use crate::common::Server; -use serde_json::json; - -#[actix_rt::test] -async fn get_unexisting_dump_status() { - let server = Server::new().await; - - let (response, code) = server.get_dump_status("foobar").await; - assert_eq!(code, 404); - - let expected_response = json!({ - "message": "Dump `foobar` not found.", - "code": "dump_not_found", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#dump_not_found" - }); - - assert_eq!(response, expected_response); -} diff --git a/meilisearch-http/tests/dumps/data.rs b/meilisearch-http/tests/dumps/data.rs new file mode 100644 index 000000000..5df09bfd1 --- /dev/null +++ b/meilisearch-http/tests/dumps/data.rs @@ -0,0 +1,73 @@ +use std::path::PathBuf; + +use manifest_dir_macros::exist_relative_path; + +pub enum GetDump { + MoviesRawV1, + MoviesWithSettingsV1, + RubyGemsWithSettingsV1, + + MoviesRawV2, + MoviesWithSettingsV2, + RubyGemsWithSettingsV2, + + MoviesRawV3, + MoviesWithSettingsV3, + RubyGemsWithSettingsV3, + + MoviesRawV4, + MoviesWithSettingsV4, + RubyGemsWithSettingsV4, + + TestV5, +} + +impl GetDump { + pub fn path(&self) -> PathBuf { + match self { + GetDump::MoviesRawV1 => { + exist_relative_path!("tests/assets/v1_v0.20.0_movies.dump").into() + } + GetDump::MoviesWithSettingsV1 => { + exist_relative_path!("tests/assets/v1_v0.20.0_movies_with_settings.dump").into() + } + GetDump::RubyGemsWithSettingsV1 => { + exist_relative_path!("tests/assets/v1_v0.20.0_rubygems_with_settings.dump").into() + } + + GetDump::MoviesRawV2 => { + exist_relative_path!("tests/assets/v2_v0.21.1_movies.dump").into() + } + GetDump::MoviesWithSettingsV2 => { + exist_relative_path!("tests/assets/v2_v0.21.1_movies_with_settings.dump").into() + } + + GetDump::RubyGemsWithSettingsV2 => { + exist_relative_path!("tests/assets/v2_v0.21.1_rubygems_with_settings.dump").into() + } + + GetDump::MoviesRawV3 => { + exist_relative_path!("tests/assets/v3_v0.24.0_movies.dump").into() + } + GetDump::MoviesWithSettingsV3 => { + exist_relative_path!("tests/assets/v3_v0.24.0_movies_with_settings.dump").into() + } + GetDump::RubyGemsWithSettingsV3 => { + exist_relative_path!("tests/assets/v3_v0.24.0_rubygems_with_settings.dump").into() + } + + GetDump::MoviesRawV4 => { + exist_relative_path!("tests/assets/v4_v0.25.2_movies.dump").into() + } + GetDump::MoviesWithSettingsV4 => { + exist_relative_path!("tests/assets/v4_v0.25.2_movies_with_settings.dump").into() + } + GetDump::RubyGemsWithSettingsV4 => { + exist_relative_path!("tests/assets/v4_v0.25.2_rubygems_with_settings.dump").into() + } + GetDump::TestV5 => { + exist_relative_path!("tests/assets/v5_v0.28.0_test_dump.dump").into() + } + } + } +} diff --git a/meilisearch-http/tests/dumps/mod.rs b/meilisearch-http/tests/dumps/mod.rs new file mode 100644 index 000000000..389f6b480 --- /dev/null +++ b/meilisearch-http/tests/dumps/mod.rs @@ -0,0 +1,677 @@ +mod data; + +use crate::common::{default_settings, GetAllDocumentsOptions, Server}; +use meilisearch_http::Opt; +use serde_json::json; + +use self::data::GetDump; + +// all the following test are ignored on windows. See #2364 +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v1() { + let temp = tempfile::tempdir().unwrap(); + + for path in [ + GetDump::MoviesRawV1.path(), + GetDump::MoviesWithSettingsV1.path(), + GetDump::RubyGemsWithSettingsV1.path(), + ] { + let options = Opt { + import_dump: Some(path), + ..default_settings(temp.path()) + }; + let error = Server::new_with_options(options) + .await + .map(|_| ()) + .unwrap_err(); + + assert_eq!(error.to_string(), "The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards."); + } +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v2_movie_raw() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::MoviesRawV2.path()), + ..default_settings(temp.path()) + }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("indexUID")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("indexUID"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"genres": 53, "id": 53, "overview": 53, "poster": 53, "release_date": 53, "title": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks, + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "limit": 20, "from": 0, "next": null }) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(100, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({"id": 100, "title": "Lock, Stock and Two Smoking Barrels", "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "genres": ["Comedy", "Crime"], "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000}) + ); + + let (document, code) = index.get_document(500, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({"id": 500, "title": "Reservoir Dogs", "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "genres": ["Crime", "Thriller"], "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + ); + + let (document, code) = index.get_document(10006, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({"id": 10006, "title": "Wild Seven", "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "genres": ["Action", "Crime", "Drama"], "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v2_movie_with_settings() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::MoviesWithSettingsV2.path()), + ..default_settings(temp.path()) + }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("indexUID")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("indexUID"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"genres": 53, "id": 53, "overview": 53, "poster": 53, "release_date": 53, "title": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks, + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "limit": 20, "from": 1, "next": null }) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(100, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) + ); + + let (document, code) = index.get_document(500, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + ); + + let (document, code) = index.get_document(10006, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v2_rubygems_with_settings() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::RubyGemsWithSettingsV2.path()), + ..default_settings(temp.path()) + }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("rubygems")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("rubygems"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"description": 53, "id": 53, "name": 53, "summary": 53, "total_downloads": 53, "version": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 }}) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks["results"][0], + json!({"uid": 92, "indexUid": "rubygems", "status": "succeeded", "type": "documentAdditionOrUpdate", "details": {"receivedDocuments": 0, "indexedDocuments": 1042}, "duration": "PT14.034672S", "enqueuedAt": "2021-09-08T08:40:31.390775Z", "startedAt": "2021-09-08T08:51:39.060642Z", "finishedAt": "2021-09-08T08:51:53.095314Z"}) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(188040, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"}) + ); + + let (document, code) = index.get_document(191940, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"}) + ); + + let (document, code) = index.get_document(159227, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v3_movie_raw() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::MoviesRawV3.path()), + ..default_settings(temp.path()) + }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("indexUID")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("indexUID"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"genres": 53, "id": 53, "overview": 53, "poster": 53, "release_date": 53, "title": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks, + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "limit": 20, "from": 0, "next": null }) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(100, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({"id": 100, "title": "Lock, Stock and Two Smoking Barrels", "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "genres": ["Comedy", "Crime"], "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000}) + ); + + let (document, code) = index.get_document(500, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({"id": 500, "title": "Reservoir Dogs", "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "genres": ["Crime", "Thriller"], "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + ); + + let (document, code) = index.get_document(10006, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({"id": 10006, "title": "Wild Seven", "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "genres": ["Action", "Crime", "Drama"], "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v3_movie_with_settings() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::MoviesWithSettingsV3.path()), + ..default_settings(temp.path()) + }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("indexUID")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("indexUID"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"genres": 53, "id": 53, "overview": 53, "poster": 53, "release_date": 53, "title": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks, + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "limit": 20, "from": 1, "next": null }) + ); + + // finally we're just going to check that we can["results"] still get a few documents by id + let (document, code) = index.get_document(100, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) + ); + + let (document, code) = index.get_document(500, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + ); + + let (document, code) = index.get_document(10006, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v3_rubygems_with_settings() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::RubyGemsWithSettingsV3.path()), + ..default_settings(temp.path()) + }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("rubygems")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("rubygems"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"description": 53, "id": 53, "name": 53, "summary": 53, "total_downloads": 53, "version": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({"displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks["results"][0], + json!({"uid": 92, "indexUid": "rubygems", "status": "succeeded", "type": "documentAdditionOrUpdate", "details": {"receivedDocuments": 0, "indexedDocuments": 1042}, "duration": "PT14.034672S", "enqueuedAt": "2021-09-08T08:40:31.390775Z", "startedAt": "2021-09-08T08:51:39.060642Z", "finishedAt": "2021-09-08T08:51:53.095314Z"}) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(188040, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"}) + ); + + let (document, code) = index.get_document(191940, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"}) + ); + + let (document, code) = index.get_document(159227, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v4_movie_raw() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::MoviesRawV4.path()), + ..default_settings(temp.path()) + }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("indexUID")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("indexUID"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"genres": 53, "id": 53, "overview": 53, "poster": 53, "release_date": 53, "title": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({ "displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks, + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "limit" : 20, "from": 0, "next": null }) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(100, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "genres": ["Comedy", "Crime"], "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000}) + ); + + let (document, code) = index.get_document(500, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 500, "title": "Reservoir Dogs", "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "genres": ["Crime", "Thriller"], "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + ); + + let (document, code) = index.get_document(10006, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 10006, "title": "Wild Seven", "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "genres": ["Action", "Crime", "Drama"], "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v4_movie_with_settings() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::MoviesWithSettingsV4.path()), + ..default_settings(temp.path()) + }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("indexUID")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("indexUID"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"genres": 53, "id": 53, "overview": 53, "poster": 53, "release_date": 53, "title": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({ "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["words", "typo", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks, + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "limit": 20, "from": 1, "next": null }) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(100, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) + ); + + let (document, code) = index.get_document(500, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + ); + + let (document, code) = index.get_document(10006, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v4_rubygems_with_settings() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::RubyGemsWithSettingsV4.path()), + ..default_settings(temp.path()) + }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("rubygems")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("rubygems"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"description": 53, "id": 53, "name": 53, "summary": 53, "total_downloads": 53, "version": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({ "displayedAttributes": ["name", "summary", "description", "version", "total_downloads"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks["results"][0], + json!({ "uid": 92, "indexUid": "rubygems", "status": "succeeded", "type": "documentAdditionOrUpdate", "details": {"receivedDocuments": 0, "indexedDocuments": 1042}, "duration": "PT14.034672S", "enqueuedAt": "2021-09-08T08:40:31.390775Z", "startedAt": "2021-09-08T08:51:39.060642Z", "finishedAt": "2021-09-08T08:51:53.095314Z"}) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(188040, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"}) + ); + + let (document, code) = index.get_document(191940, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"}) + ); + + let (document, code) = index.get_document(159227, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v5() { + let temp = tempfile::tempdir().unwrap(); + + let options = Opt { + import_dump: Some(GetDump::TestV5.path()), + ..default_settings(temp.path()) + }; + let mut server = Server::new_auth_with_options(options, temp).await; + server.use_api_key("MASTER_KEY"); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200, "{indexes}"); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 2); + assert_eq!(indexes["results"][0]["uid"], json!("test")); + assert_eq!(indexes["results"][1]["uid"], json!("test2")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let expected_stats = json!({ + "numberOfDocuments": 10, + "isIndexing": false, + "fieldDistribution": { + "cast": 10, + "director": 10, + "genres": 10, + "id": 10, + "overview": 10, + "popularity": 10, + "poster_path": 10, + "producer": 10, + "production_companies": 10, + "release_date": 10, + "tagline": 10, + "title": 10, + "vote_average": 10, + "vote_count": 10 + } + }); + + let index1 = server.index("test"); + let index2 = server.index("test2"); + + let (stats, code) = index1.stats().await; + assert_eq!(code, 200); + assert_eq!(stats, expected_stats); + + let (docs, code) = index2 + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 200); + assert_eq!(docs["results"].as_array().unwrap().len(), 10); + let (docs, code) = index1 + .get_all_documents(GetAllDocumentsOptions::default()) + .await; + assert_eq!(code, 200); + assert_eq!(docs["results"].as_array().unwrap().len(), 10); + + let (stats, code) = index2.stats().await; + assert_eq!(code, 200); + assert_eq!(stats, expected_stats); + + let (keys, code) = server.list_api_keys().await; + assert_eq!(code, 200); + let key = &keys["results"][0]; + + assert_eq!(key["name"], "my key"); +} diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index 0e134600e..a1c508e1f 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -102,7 +102,7 @@ async fn error_create_with_invalid_index_uid() { let (response, code) = index.create(None).await; let expected_response = json!({ - "message": "`test test#!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "invalid index uid `test test#!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 0674d0afd..f3cdf6631 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -52,10 +52,10 @@ async fn loop_delete_add_documents() { let mut tasks = Vec::new(); for _ in 0..50 { let (response, code) = index.add_documents(documents.clone(), None).await; - tasks.push(response["uid"].as_u64().unwrap()); + tasks.push(response["taskUid"].as_u64().unwrap()); assert_eq!(code, 202, "{}", response); let (response, code) = index.delete().await; - tasks.push(response["uid"].as_u64().unwrap()); + tasks.push(response["taskUid"].as_u64().unwrap()); assert_eq!(code, 202, "{}", response); } diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index 924f603df..91cb1a6d5 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -16,12 +16,11 @@ async fn create_and_get_index() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); assert_eq!(response["createdAt"], response["updatedAt"]); assert_eq!(response["primaryKey"], Value::Null); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } #[actix_rt::test] @@ -45,10 +44,10 @@ async fn error_get_unexisting_index() { #[actix_rt::test] async fn no_index_return_empty_list() { let server = Server::new().await; - let (response, code) = server.list_indexes().await; + let (response, code) = server.list_indexes(None, None).await; assert_eq!(code, 200); - assert!(response.is_array()); - assert!(response.as_array().unwrap().is_empty()); + assert!(response["results"].is_array()); + assert!(response["results"].as_array().unwrap().is_empty()); } #[actix_rt::test] @@ -59,10 +58,10 @@ async fn list_multiple_indexes() { server.index("test").wait_task(1).await; - let (response, code) = server.list_indexes().await; + let (response, code) = server.list_indexes(None, None).await; assert_eq!(code, 200); - assert!(response.is_array()); - let arr = response.as_array().unwrap(); + assert!(response["results"].is_array()); + let arr = response["results"].as_array().unwrap(); assert_eq!(arr.len(), 2); assert!(arr .iter() @@ -72,6 +71,118 @@ async fn list_multiple_indexes() { .any(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key")); } +#[actix_rt::test] +async fn get_and_paginate_indexes() { + let server = Server::new().await; + const NB_INDEXES: usize = 50; + for i in 0..NB_INDEXES { + server.index(&format!("test_{i:02}")).create(None).await; + server + .index(&format!("test_{i:02}")) + .wait_task(i as u64) + .await; + } + + // basic + let (response, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["total"], json!(NB_INDEXES)); + assert!(response["results"].is_array()); + let arr = response["results"].as_array().unwrap(); + assert_eq!(arr.len(), 20); + // ensuring we get all the indexes in the alphabetical order + assert!((0..20) + .map(|idx| format!("test_{idx:02}")) + .zip(arr) + .all(|(expected, entry)| entry["uid"] == expected)); + + // with an offset + let (response, code) = server.list_indexes(Some(15), None).await; + assert_eq!(code, 200); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["offset"], json!(15)); + assert_eq!(response["total"], json!(NB_INDEXES)); + assert!(response["results"].is_array()); + let arr = response["results"].as_array().unwrap(); + assert_eq!(arr.len(), 20); + assert!((15..35) + .map(|idx| format!("test_{idx:02}")) + .zip(arr) + .all(|(expected, entry)| entry["uid"] == expected)); + + // with an offset and not enough elements + let (response, code) = server.list_indexes(Some(45), None).await; + assert_eq!(code, 200); + assert_eq!(response["limit"], json!(20)); + assert_eq!(response["offset"], json!(45)); + assert_eq!(response["total"], json!(NB_INDEXES)); + assert!(response["results"].is_array()); + let arr = response["results"].as_array().unwrap(); + assert_eq!(arr.len(), 5); + assert!((45..50) + .map(|idx| format!("test_{idx:02}")) + .zip(arr) + .all(|(expected, entry)| entry["uid"] == expected)); + + // with a limit lower than the default + let (response, code) = server.list_indexes(None, Some(5)).await; + assert_eq!(code, 200); + assert_eq!(response["limit"], json!(5)); + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["total"], json!(NB_INDEXES)); + assert!(response["results"].is_array()); + let arr = response["results"].as_array().unwrap(); + assert_eq!(arr.len(), 5); + assert!((0..5) + .map(|idx| format!("test_{idx:02}")) + .zip(arr) + .all(|(expected, entry)| entry["uid"] == expected)); + + // with a limit higher than the default + let (response, code) = server.list_indexes(None, Some(40)).await; + assert_eq!(code, 200); + assert_eq!(response["limit"], json!(40)); + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["total"], json!(NB_INDEXES)); + assert!(response["results"].is_array()); + let arr = response["results"].as_array().unwrap(); + assert_eq!(arr.len(), 40); + assert!((0..40) + .map(|idx| format!("test_{idx:02}")) + .zip(arr) + .all(|(expected, entry)| entry["uid"] == expected)); + + // with a limit higher than the default + let (response, code) = server.list_indexes(None, Some(80)).await; + assert_eq!(code, 200); + assert_eq!(response["limit"], json!(80)); + assert_eq!(response["offset"], json!(0)); + assert_eq!(response["total"], json!(NB_INDEXES)); + assert!(response["results"].is_array()); + let arr = response["results"].as_array().unwrap(); + assert_eq!(arr.len(), 50); + assert!((0..50) + .map(|idx| format!("test_{idx:02}")) + .zip(arr) + .all(|(expected, entry)| entry["uid"] == expected)); + + // with a limit and an offset + let (response, code) = server.list_indexes(Some(20), Some(10)).await; + assert_eq!(code, 200); + assert_eq!(response["limit"], json!(10)); + assert_eq!(response["offset"], json!(20)); + assert_eq!(response["total"], json!(NB_INDEXES)); + assert!(response["results"].is_array()); + let arr = response["results"].as_array().unwrap(); + assert_eq!(arr.len(), 10); + assert!((20..30) + .map(|idx| format!("test_{idx:02}")) + .zip(arr) + .all(|(expected, entry)| entry["uid"] == expected)); +} + #[actix_rt::test] async fn get_invalid_index_uid() { let server = Server::new().await; diff --git a/meilisearch-http/tests/index/stats.rs b/meilisearch-http/tests/index/stats.rs index 555c7311a..f55998998 100644 --- a/meilisearch-http/tests/index/stats.rs +++ b/meilisearch-http/tests/index/stats.rs @@ -35,7 +35,7 @@ async fn stats() { let (response, code) = index.add_documents(documents, None).await; assert_eq!(code, 202); - assert_eq!(response["uid"], 1); + assert_eq!(response["taskUid"], 1); index.wait_task(1).await; diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index 1896f731f..48fde5608 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -21,7 +21,6 @@ async fn update_primary_key() { assert_eq!(code, 200); assert_eq!(response["uid"], "test"); - assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); @@ -32,7 +31,7 @@ async fn update_primary_key() { assert!(created_at < updated_at); assert_eq!(response["primaryKey"], "primary"); - assert_eq!(response.as_object().unwrap().len(), 5); + assert_eq!(response.as_object().unwrap().len(), 4); } #[actix_rt::test] diff --git a/meilisearch-http/tests/integration.rs b/meilisearch-http/tests/integration.rs index 45b632520..25b4e49b6 100644 --- a/meilisearch-http/tests/integration.rs +++ b/meilisearch-http/tests/integration.rs @@ -2,6 +2,7 @@ mod auth; mod common; mod dashboard; mod documents; +mod dumps; mod index; mod search; mod settings; diff --git a/meilisearch-http/tests/search/errors.rs b/meilisearch-http/tests/search/errors.rs index 500825364..98da0495a 100644 --- a/meilisearch-http/tests/search/errors.rs +++ b/meilisearch-http/tests/search/errors.rs @@ -45,26 +45,18 @@ async fn search_invalid_highlight_and_crop_tags() { for field in fields { // object - index - .search( - json!({field.to_string(): {"marker": ""}}), - |response, code| { - assert_eq!(code, 400, "field {} passing object: {}", &field, response); - assert_eq!(response["code"], "bad_request"); - }, - ) + let (response, code) = index + .search_post(json!({field.to_string(): {"marker": ""}})) .await; + assert_eq!(code, 400, "field {} passing object: {}", &field, response); + assert_eq!(response["code"], "bad_request"); // array - index - .search( - json!({field.to_string(): ["marker", ""]}), - |response, code| { - assert_eq!(code, 400, "field {} passing array: {}", &field, response); - assert_eq!(response["code"], "bad_request"); - }, - ) + let (response, code) = index + .search_post(json!({field.to_string(): ["marker", ""]})) .await; + assert_eq!(code, 400, "field {} passing array: {}", &field, response); + assert_eq!(response["code"], "bad_request"); } } @@ -115,7 +107,7 @@ async fn filter_invalid_syntax_array() { "link": "https://docs.meilisearch.com/errors#invalid_filter" }); index - .search(json!({"filter": [["title & Glass"]]}), |response, code| { + .search(json!({"filter": ["title & Glass"]}), |response, code| { assert_eq!(response, expected_response); assert_eq!(code, 400); }) @@ -172,7 +164,7 @@ async fn filter_invalid_attribute_array() { "link": "https://docs.meilisearch.com/errors#invalid_filter" }); index - .search(json!({"filter": [["many = Glass"]]}), |response, code| { + .search(json!({"filter": ["many = Glass"]}), |response, code| { assert_eq!(response, expected_response); assert_eq!(code, 400); }) @@ -226,7 +218,7 @@ async fn filter_reserved_geo_attribute_array() { "link": "https://docs.meilisearch.com/errors#invalid_filter" }); index - .search(json!({"filter": [["_geo = Glass"]]}), |response, code| { + .search(json!({"filter": ["_geo = Glass"]}), |response, code| { assert_eq!(response, expected_response); assert_eq!(code, 400); }) @@ -281,7 +273,7 @@ async fn filter_reserved_attribute_array() { }); index .search( - json!({"filter": [["_geoDistance = Glass"]]}), + json!({"filter": ["_geoDistance = Glass"]}), |response, code| { assert_eq!(response, expected_response); assert_eq!(code, 400); diff --git a/meilisearch-http/tests/search/formatted.rs b/meilisearch-http/tests/search/formatted.rs index 13b8a07d8..7303a7154 100644 --- a/meilisearch-http/tests/search/formatted.rs +++ b/meilisearch-http/tests/search/formatted.rs @@ -15,83 +15,100 @@ async fn formatted_contain_wildcard() { index.add_documents(documents, None).await; index.wait_task(1).await; - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["father", "mother"], "attributesToHighlight": ["father", "mother", "*"], "attributesToCrop": ["doggos"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "id": "852", - "cattos": "pesti", - } - }) - ); + index.search(json!({ "q": "pesti", "attributesToRetrieve": ["father", "mother"], "attributesToHighlight": ["father", "mother", "*"], "attributesToCrop": ["doggos"], "showMatchesPosition": true }), + |response, code| + { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "id": "852", + "cattos": "pesti", + }, + "_matchesPosition": {"cattos": [{"start": 0, "length": 5}]}, + }) + ); + } + ) + .await; - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["*"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "cattos": "pesti", - }) - ); - - let (response, code) = index - .search_post( - json!({ "q": "pesti", "attributesToRetrieve": ["*"], "attributesToHighlight": ["id"] }), + index + .search( + json!({ "q": "pesti", "attributesToRetrieve": ["*"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "cattos": "pesti", + }) + ); + }, ) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "cattos": "pesti", - "_formatted": { - "id": "852", - "cattos": "pesti", - } - }) - ); - let (response, code) = index - .search_post( + index + .search( + json!({ "q": "pesti", "attributesToRetrieve": ["*"], "attributesToHighlight": ["id"], "showMatchesPosition": true }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "cattos": "pesti", + "_formatted": { + "id": "852", + "cattos": "pesti", + }, + "_matchesPosition": {"cattos": [{"start": 0, "length": 5}]}, + }) + ); + } + ) + .await; + + index + .search( json!({ "q": "pesti", "attributesToRetrieve": ["*"], "attributesToCrop": ["*"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "cattos": "pesti", + "_formatted": { + "id": "852", + "cattos": "pesti", + } + }) + ); + }, ) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "cattos": "pesti", - "_formatted": { - "id": "852", - "cattos": "pesti", - } - }) - ); - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToCrop": ["*"] })) + index + .search( + json!({ "q": "pesti", "attributesToCrop": ["*"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "cattos": "pesti", + "_formatted": { + "id": "852", + "cattos": "pesti", + } + }) + ); + }, + ) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "cattos": "pesti", - "_formatted": { - "id": "852", - "cattos": "pesti", - } - }) - ); } #[actix_rt::test] @@ -103,87 +120,122 @@ async fn format_nested() { index.add_documents(documents, None).await; index.wait_task(0).await; - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["doggos"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "doggos": [ - { - "name": "bobby", - "age": 2, - }, - { - "name": "buddy", - "age": 4, - }, - ], - }) - ); - - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "doggos": [ - { - "name": "bobby", - }, - { - "name": "buddy", - }, - ], - }) - ); - - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.name"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "doggos": [ - { - "name": "bobby", - }, - { - "name": "buddy", - }, - ], + index + .search( + json!({ "q": "pesti", "attributesToRetrieve": ["doggos"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "doggos": [ + { + "name": "bobby", + "age": 2, + }, + { + "name": "buddy", + "age": 4, + }, + ], + }) + ); }, - }) - ); - - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToCrop": ["doggos.name"] })) + ) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "doggos": [ - { - "name": "bobby", - }, - { - "name": "buddy", - }, - ], + + index + .search( + json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "doggos": [ + { + "name": "bobby", + }, + { + "name": "buddy", + }, + ], + }) + ); }, - }) - ); - - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"], "attributesToHighlight": ["doggos.age"] })) + ) .await; + + index + .search( + json!({ "q": "bobby", "attributesToRetrieve": ["doggos.name"], "showMatchesPosition": true }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "doggos": [ + { + "name": "bobby", + }, + { + "name": "buddy", + }, + ], + "_matchesPosition": {"doggos.name": [{"start": 0, "length": 5}]}, + }) + ); + } + ) + .await; + + index + .search(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.name"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "doggos": [ + { + "name": "bobby", + }, + { + "name": "buddy", + }, + ], + }, + }) + ); + }) + .await; + + index + .search(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToCrop": ["doggos.name"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "doggos": [ + { + "name": "bobby", + }, + { + "name": "buddy", + }, + ], + }, + }) + ); + }) + .await; + + index + .search(json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"], "attributesToHighlight": ["doggos.age"] }), + |response, code| { assert_eq!(code, 200, "{}", response); assert_eq!( response["hits"][0], @@ -210,11 +262,13 @@ async fn format_nested() { }, }) ); - - let (response, code) = index - .search_post(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.age"], "attributesToCrop": ["doggos.name"] })) + }) .await; - assert_eq!(code, 200, "{}", response); + + index + .search(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.age"], "attributesToCrop": ["doggos.name"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); assert_eq!( response["hits"][0], json!({ @@ -232,6 +286,9 @@ async fn format_nested() { }, }) ); + } + ) + .await; } #[actix_rt::test] @@ -248,9 +305,9 @@ async fn displayedattr_2_smol() { index.add_documents(documents, None).await; index.wait_task(1).await; - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["father", "id"], "attributesToHighlight": ["mother"], "attributesToCrop": ["cattos"] })) - .await; + index + .search(json!({ "attributesToRetrieve": ["father", "id"], "attributesToHighlight": ["mother"], "attributesToCrop": ["cattos"] }), + |response, code| { assert_eq!(code, 200, "{}", response); assert_eq!( response["hits"][0], @@ -258,119 +315,157 @@ async fn displayedattr_2_smol() { "id": 852, }) ); - - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["id"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToHighlight": ["id"] })) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "_formatted": { - "id": "852", - } - }) - ); - let (response, code) = index - .search_post(json!({ "attributesToCrop": ["id"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "_formatted": { - "id": "852", - } - }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToHighlight": ["id"], "attributesToCrop": ["id"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - "_formatted": { - "id": "852", - } - }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToHighlight": ["cattos"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToCrop": ["cattos"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "id": 852, - }) - ); - - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["cattos"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!(response["hits"][0], json!({})); - - let (response, code) = index - .search_post( - json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["cattos"], "attributesToCrop": ["cattos"] }), + index + .search( + json!({ "attributesToRetrieve": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + }) + ); + }, ) .await; + + index + .search( + json!({ "attributesToHighlight": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "_formatted": { + "id": "852", + } + }) + ); + }, + ) + .await; + + index + .search(json!({ "attributesToCrop": ["id"] }), |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "_formatted": { + "id": "852", + } + }) + ); + }) + .await; + + index + .search( + json!({ "attributesToHighlight": ["id"], "attributesToCrop": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + "_formatted": { + "id": "852", + } + }) + ); + }, + ) + .await; + + index + .search( + json!({ "attributesToHighlight": ["cattos"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + }) + ); + }, + ) + .await; + + index + .search( + json!({ "attributesToCrop": ["cattos"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "id": 852, + }) + ); + }, + ) + .await; + + index + .search( + json!({ "attributesToRetrieve": ["cattos"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"][0], json!({})); + }, + ) + .await; + + index + .search( + json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["cattos"], "attributesToCrop": ["cattos"] }), + |response, code| { assert_eq!(code, 200, "{}", response); assert_eq!(response["hits"][0], json!({})); - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["id"] })) - .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "id": "852", } - }) - ); + ) + .await; - let (response, code) = index - .search_post(json!({ "attributesToRetrieve": ["cattos"], "attributesToCrop": ["id"] })) + index + .search( + json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "id": "852", + } + }) + ); + }, + ) + .await; + + index + .search( + json!({ "attributesToRetrieve": ["cattos"], "attributesToCrop": ["id"] }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!( + response["hits"][0], + json!({ + "_formatted": { + "id": "852", + } + }) + ); + }, + ) .await; - assert_eq!(code, 200, "{}", response); - assert_eq!( - response["hits"][0], - json!({ - "_formatted": { - "id": "852", - } - }) - ); } diff --git a/meilisearch-http/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs index d9b36e85d..b9c187783 100644 --- a/meilisearch-http/tests/search/mod.rs +++ b/meilisearch-http/tests/search/mod.rs @@ -420,11 +420,11 @@ async fn search_facet_distribution() { index .search( json!({ - "facetsDistribution": ["title"] + "facets": ["title"] }), |response, code| { assert_eq!(code, 200, "{}", response); - let dist = response["facetsDistribution"].as_object().unwrap(); + let dist = response["facetDistribution"].as_object().unwrap(); assert_eq!(dist.len(), 1); assert!(dist.get("title").is_some()); }, @@ -445,12 +445,12 @@ async fn search_facet_distribution() { index .search( json!({ - // "facetsDistribution": ["father", "doggos.name"] - "facetsDistribution": ["father"] + // "facets": ["father", "doggos.name"] + "facets": ["father"] }), |response, code| { assert_eq!(code, 200, "{}", response); - let dist = response["facetsDistribution"].as_object().unwrap(); + let dist = response["facetDistribution"].as_object().unwrap(); assert_eq!(dist.len(), 1); assert_eq!( dist["father"], @@ -474,11 +474,11 @@ async fn search_facet_distribution() { index .search( json!({ - "facetsDistribution": ["doggos.name"] + "facets": ["doggos.name"] }), |response, code| { assert_eq!(code, 200, "{}", response); - let dist = response["facetsDistribution"].as_object().unwrap(); + let dist = response["facetDistribution"].as_object().unwrap(); assert_eq!(dist.len(), 1); assert_eq!( dist["doggos.name"], @@ -491,12 +491,11 @@ async fn search_facet_distribution() { index .search( json!({ - "facetsDistribution": ["doggos"] + "facets": ["doggos"] }), |response, code| { assert_eq!(code, 200, "{}", response); - let dist = response["facetsDistribution"].as_object().unwrap(); - dbg!(&dist); + let dist = response["facetDistribution"].as_object().unwrap(); assert_eq!(dist.len(), 3); assert_eq!( dist["doggos.name"], @@ -566,6 +565,36 @@ async fn placeholder_search_is_hard_limited() { }, ) .await; + + index + .update_settings(json!({ "pagination": { "maxTotalHits": 10_000 } })) + .await; + index.wait_task(1).await; + + index + .search( + json!({ + "limit": 1500, + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1200); + }, + ) + .await; + + index + .search( + json!({ + "offset": 1000, + "limit": 400, + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 200); + }, + ) + .await; } #[actix_rt::test] @@ -605,4 +634,85 @@ async fn search_is_hard_limited() { }, ) .await; + + index + .update_settings(json!({ "pagination": { "maxTotalHits": 10_000 } })) + .await; + index.wait_task(1).await; + + index + .search( + json!({ + "q": "unique", + "limit": 1500, + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1200); + }, + ) + .await; + + index + .search( + json!({ + "q": "unique", + "offset": 1000, + "limit": 400, + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 200); + }, + ) + .await; +} + +#[actix_rt::test] +async fn faceting_max_values_per_facet() { + let server = Server::new().await; + let index = server.index("test"); + + index + .update_settings(json!({ "filterableAttributes": ["number"] })) + .await; + + let documents: Vec<_> = (0..10_000) + .map(|id| json!({ "id": id, "number": id * 10 })) + .collect(); + index.add_documents(json!(documents), None).await; + index.wait_task(1).await; + + index + .search( + json!({ + "facets": ["number"] + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + let numbers = response["facetDistribution"]["number"].as_object().unwrap(); + assert_eq!(numbers.len(), 100); + }, + ) + .await; + + index + .update_settings(json!({ "faceting": { "maxValuesPerFacet": 10_000 } })) + .await; + index.wait_task(2).await; + + index + .search( + json!({ + "facets": ["number"] + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + let numbers = dbg!(&response)["facetDistribution"]["number"] + .as_object() + .unwrap(); + assert_eq!(numbers.len(), 10_000); + }, + ) + .await; } diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 98b4f9558..9d10b7820 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -24,6 +24,18 @@ static DEFAULT_SETTINGS_VALUES: Lazy> = Lazy::new(| ); map.insert("stop_words", json!([])); map.insert("synonyms", json!({})); + map.insert( + "faceting", + json!({ + "maxValuesPerFacet": json!(100), + }), + ); + map.insert( + "pagination", + json!({ + "maxTotalHits": json!(1000), + }), + ); map }); @@ -43,7 +55,7 @@ async fn get_settings() { let (response, code) = index.settings().await; assert_eq!(code, 200); let settings = response.as_object().unwrap(); - assert_eq!(settings.keys().len(), 9); + assert_eq!(settings.keys().len(), 11); assert_eq!(settings["displayedAttributes"], json!(["*"])); assert_eq!(settings["searchableAttributes"], json!(["*"])); assert_eq!(settings["filterableAttributes"], json!([])); @@ -61,6 +73,18 @@ async fn get_settings() { ]) ); assert_eq!(settings["stopWords"], json!([])); + assert_eq!( + settings["faceting"], + json!({ + "maxValuesPerFacet": 100, + }) + ); + assert_eq!( + settings["pagination"], + json!({ + "maxTotalHits": 1000, + }) + ); } #[actix_rt::test] @@ -122,7 +146,7 @@ async fn reset_all_settings() { let (response, code) = index.add_documents(documents, None).await; assert_eq!(code, 202); - assert_eq!(response["uid"], 0); + assert_eq!(response["taskUid"], 0); index.wait_task(0).await; index @@ -179,7 +203,7 @@ async fn error_update_setting_unexisting_index_invalid_uid() { assert_eq!(code, 400); let expected = json!({ - "message": "`test##! ` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "invalid index uid `test##! `, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid"}); @@ -188,7 +212,7 @@ async fn error_update_setting_unexisting_index_invalid_uid() { } macro_rules! test_setting_routes { - ($($setting:ident), *) => { + ($($setting:ident $write_method:ident), *) => { $( mod $setting { use crate::common::Server; @@ -214,7 +238,7 @@ macro_rules! test_setting_routes { .chars() .map(|c| if c == '_' { '-' } else { c }) .collect::()); - let (response, code) = server.service.post(url, serde_json::Value::Null).await; + let (response, code) = server.service.$write_method(url, serde_json::Value::Null).await; assert_eq!(code, 202, "{}", response); server.index("").wait_task(0).await; let (response, code) = server.index("test").get().await; @@ -258,13 +282,15 @@ macro_rules! test_setting_routes { } test_setting_routes!( - filterable_attributes, - displayed_attributes, - searchable_attributes, - distinct_attribute, - stop_words, - ranking_rules, - synonyms + filterable_attributes put, + displayed_attributes put, + searchable_attributes put, + distinct_attribute put, + stop_words put, + ranking_rules put, + synonyms put, + pagination patch, + faceting patch ); #[actix_rt::test] @@ -283,7 +309,7 @@ async fn error_set_invalid_ranking_rules() { assert_eq!(response["status"], "failed"); let expected_error = json!({ - "message": r#"`manyTheFish` ranking rule is invalid. Valid ranking rules are Words, Typo, Sort, Proximity, Attribute, Exactness and custom ranking rules."#, + "message": r#"`manyTheFish` ranking rule is invalid. Valid ranking rules are words, typo, sort, proximity, attribute, exactness and custom ranking rules."#, "code": "invalid_ranking_rule", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_ranking_rule" diff --git a/meilisearch-http/tests/snapshot/mod.rs b/meilisearch-http/tests/snapshot/mod.rs index 5c626a888..27ff838e1 100644 --- a/meilisearch-http/tests/snapshot/mod.rs +++ b/meilisearch-http/tests/snapshot/mod.rs @@ -41,7 +41,7 @@ async fn perform_snapshot() { ..default_settings(temp.path()) }; - let server = Server::new_with_options(options).await; + let server = Server::new_with_options(options).await.unwrap(); let index = server.index("test"); index @@ -67,10 +67,10 @@ async fn perform_snapshot() { ..default_settings(temp.path()) }; - let snapshot_server = Server::new_with_options(options).await; + let snapshot_server = Server::new_with_options(options).await.unwrap(); verify_snapshot!(server, snapshot_server, |server| => - server.list_indexes(), + server.list_indexes(None, None), // for some reason the db sizes differ. this may be due to the compaction options we have // set when performing the snapshot //server.stats(), diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index b9d185ca3..0629c2e29 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -54,7 +54,7 @@ async fn stats() { let (response, code) = index.add_documents(documents, None).await; assert_eq!(code, 202, "{}", response); - assert_eq!(response["uid"], 1); + assert_eq!(response["taskUid"], 1); index.wait_task(1).await; diff --git a/meilisearch-http/tests/tasks/mod.rs b/meilisearch-http/tests/tasks/mod.rs index 167b7b05f..9d0940562 100644 --- a/meilisearch-http/tests/tasks/mod.rs +++ b/meilisearch-http/tests/tasks/mod.rs @@ -3,22 +3,6 @@ use serde_json::json; use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; -#[actix_rt::test] -async fn error_get_task_unexisting_index() { - let server = Server::new().await; - let (response, code) = server.service.get("/indexes/test/tasks").await; - - let expected_response = json!({ - "message": "Index `test` not found.", - "code": "index_not_found", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#index_not_found" - }); - - assert_eq!(response, expected_response); - assert_eq!(code, 404); -} - #[actix_rt::test] async fn error_get_unexisting_task_status() { let server = Server::new().await; @@ -58,22 +42,6 @@ async fn get_task_status() { // TODO check resonse format, as per #48 } -#[actix_rt::test] -async fn error_list_tasks_unexisting_index() { - let server = Server::new().await; - let (response, code) = server.index("test").list_tasks().await; - - let expected_response = json!({ - "message": "Index `test` not found.", - "code": "index_not_found", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#index_not_found" - }); - - assert_eq!(response, expected_response); - assert_eq!(code, 404); -} - #[actix_rt::test] async fn list_tasks() { let server = Server::new().await; @@ -91,10 +59,140 @@ async fn list_tasks() { assert_eq!(response["results"].as_array().unwrap().len(), 2); } +#[actix_rt::test] +async fn list_tasks_with_star_filters() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + index.wait_task(0).await; + index + .add_documents( + serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), + None, + ) + .await; + let (response, code) = index.service.get("/tasks?indexUid=test").await; + assert_eq!(code, 200); + assert_eq!(response["results"].as_array().unwrap().len(), 2); + + let (response, code) = index.service.get("/tasks?indexUid=*").await; + assert_eq!(code, 200); + assert_eq!(response["results"].as_array().unwrap().len(), 2); + + let (response, code) = index.service.get("/tasks?indexUid=*,pasteque").await; + assert_eq!(code, 200); + assert_eq!(response["results"].as_array().unwrap().len(), 2); + + let (response, code) = index.service.get("/tasks?type=*").await; + assert_eq!(code, 200); + assert_eq!(response["results"].as_array().unwrap().len(), 2); + + let (response, code) = index + .service + .get("/tasks?type=*,documentAdditionOrUpdate&status=*") + .await; + assert_eq!(code, 200, "{:?}", response); + assert_eq!(response["results"].as_array().unwrap().len(), 2); + + let (response, code) = index + .service + .get("/tasks?type=*,documentAdditionOrUpdate&status=*,failed&indexUid=test") + .await; + assert_eq!(code, 200, "{:?}", response); + assert_eq!(response["results"].as_array().unwrap().len(), 2); + + let (response, code) = index + .service + .get("/tasks?type=*,documentAdditionOrUpdate&status=*,failed&indexUid=test,*") + .await; + assert_eq!(code, 200, "{:?}", response); + assert_eq!(response["results"].as_array().unwrap().len(), 2); +} + +#[actix_rt::test] +async fn list_tasks_status_filtered() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + index.wait_task(0).await; + index + .add_documents( + serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), + None, + ) + .await; + + let (response, code) = index.filtered_tasks(&[], &["succeeded"]).await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["results"].as_array().unwrap().len(), 1); + + // We can't be sure that the update isn't already processed so we can't test this + // let (response, code) = index.filtered_tasks(&[], &["processing"]).await; + // assert_eq!(code, 200, "{}", response); + // assert_eq!(response["results"].as_array().unwrap().len(), 1); + + index.wait_task(1).await; + + let (response, code) = index.filtered_tasks(&[], &["succeeded"]).await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["results"].as_array().unwrap().len(), 2); +} + +#[actix_rt::test] +async fn list_tasks_type_filtered() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + index.wait_task(0).await; + index + .add_documents( + serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), + None, + ) + .await; + + let (response, code) = index.filtered_tasks(&["indexCreation"], &[]).await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["results"].as_array().unwrap().len(), 1); + + let (response, code) = index + .filtered_tasks(&["indexCreation", "documentAdditionOrUpdate"], &[]) + .await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["results"].as_array().unwrap().len(), 2); +} + +#[actix_rt::test] +async fn list_tasks_status_and_type_filtered() { + let server = Server::new().await; + let index = server.index("test"); + index.create(None).await; + index.wait_task(0).await; + index + .add_documents( + serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), + None, + ) + .await; + + let (response, code) = index.filtered_tasks(&["indexCreation"], &["failed"]).await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["results"].as_array().unwrap().len(), 0); + + let (response, code) = index + .filtered_tasks( + &["indexCreation", "documentAdditionOrUpdate"], + &["succeeded", "processing", "enqueued"], + ) + .await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["results"].as_array().unwrap().len(), 2); +} + macro_rules! assert_valid_summarized_task { ($response:expr, $task_type:literal, $index:literal) => {{ assert_eq!($response.as_object().unwrap().len(), 5); - assert!($response["uid"].as_u64().is_some()); + assert!($response["taskUid"].as_u64().is_some()); assert_eq!($response["indexUid"], $index); assert_eq!($response["status"], "enqueued"); assert_eq!($response["type"], $task_type); @@ -119,16 +217,16 @@ async fn test_summarized_task_view() { assert_valid_summarized_task!(response, "settingsUpdate", "test"); let (response, _) = index.update_documents(json!([{"id": 1}]), None).await; - assert_valid_summarized_task!(response, "documentPartial", "test"); + assert_valid_summarized_task!(response, "documentAdditionOrUpdate", "test"); let (response, _) = index.add_documents(json!([{"id": 1}]), None).await; - assert_valid_summarized_task!(response, "documentAddition", "test"); + assert_valid_summarized_task!(response, "documentAdditionOrUpdate", "test"); let (response, _) = index.delete_document(1).await; assert_valid_summarized_task!(response, "documentDeletion", "test"); let (response, _) = index.clear_all_documents().await; - assert_valid_summarized_task!(response, "clearAll", "test"); + assert_valid_summarized_task!(response, "documentDeletion", "test"); let (response, _) = index.delete().await; assert_valid_summarized_task!(response, "indexDeletion", "test"); diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index 3d3062e65..3d5505e5f 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -1,10 +1,8 @@ [package] name = "meilisearch-lib" -version = "0.27.2" +version = "0.28.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] actix-web = { version = "4.0.1", default-features = false } anyhow = { version = "1.0.56", features = ["backtrace"] } @@ -29,8 +27,8 @@ itertools = "0.10.3" lazy_static = "1.4.0" log = "0.4.14" meilisearch-auth = { path = "../meilisearch-auth" } -meilisearch-error = { path = "../meilisearch-error" } -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.26.6" } +meilisearch-types = { path = "../meilisearch-types" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.31.1" } mime = "0.3.16" num_cpus = "1.13.1" obkv = "0.2.0" @@ -41,6 +39,7 @@ rand = "0.8.5" rayon = "1.5.1" regex = "1.5.5" reqwest = { version = "0.11.9", features = ["json", "rustls-tls"], default-features = false, optional = true } +roaring = "0.9.0" rustls = "0.20.4" serde = { version = "1.0.136", features = ["derive"] } serde_json = { version = "1.0.79", features = ["preserve_order"] } @@ -52,15 +51,15 @@ tempfile = "3.3.0" thiserror = "1.0.30" time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } tokio = { version = "1.17.0", features = ["full"] } -uuid = { version = "0.8.2", features = ["serde"] } +uuid = { version = "1.1.2", features = ["serde", "v4"] } walkdir = "2.3.2" whoami = { version = "1.2.1", optional = true } [dev-dependencies] actix-rt = "2.7.0" -meilisearch-error = { path = "../meilisearch-error", features = ["test-traits"] } +meilisearch-types = { path = "../meilisearch-types", features = ["test-traits"] } mockall = "0.11.0" -nelson = { git = "https://github.com/MarinPostma/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"} +nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"} paste = "1.0.6" proptest = "1.0.0" proptest-derive = "0.3.0" diff --git a/meilisearch-lib/src/document_formats.rs b/meilisearch-lib/src/document_formats.rs index 93c47afe8..de3d7f5d5 100644 --- a/meilisearch-lib/src/document_formats.rs +++ b/meilisearch-lib/src/document_formats.rs @@ -2,7 +2,8 @@ use std::borrow::Borrow; use std::fmt::{self, Debug, Display}; use std::io::{self, BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write}; -use meilisearch_error::{internal_error, Code, ErrorCode}; +use meilisearch_types::error::{Code, ErrorCode}; +use meilisearch_types::internal_error; use milli::documents::DocumentBatchBuilder; type Result = std::result::Result; diff --git a/meilisearch-lib/src/index_controller/dump_actor/compat/mod.rs b/meilisearch-lib/src/dump/compat/mod.rs similarity index 97% rename from meilisearch-lib/src/index_controller/dump_actor/compat/mod.rs rename to meilisearch-lib/src/dump/compat/mod.rs index 93f3f9dd7..9abac24c7 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/compat/mod.rs +++ b/meilisearch-lib/src/dump/compat/mod.rs @@ -1,5 +1,6 @@ pub mod v2; pub mod v3; +pub mod v4; /// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name. pub fn asc_ranking_rule(text: &str) -> Option<&str> { diff --git a/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs b/meilisearch-lib/src/dump/compat/v2.rs similarity index 99% rename from meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs rename to meilisearch-lib/src/dump/compat/v2.rs index a30e24794..364d894c4 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs +++ b/meilisearch-lib/src/dump/compat/v2.rs @@ -1,5 +1,5 @@ use anyhow::bail; -use meilisearch_error::Code; +use meilisearch_types::error::Code; use milli::update::IndexDocumentsMethod; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; diff --git a/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs b/meilisearch-lib/src/dump/compat/v3.rs similarity index 91% rename from meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs rename to meilisearch-lib/src/dump/compat/v3.rs index 7cd670bad..61e31eccd 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs +++ b/meilisearch-lib/src/dump/compat/v3.rs @@ -1,12 +1,13 @@ -use meilisearch_error::{Code, ResponseError}; +use meilisearch_types::error::{Code, ResponseError}; +use meilisearch_types::index_uid::IndexUid; use milli::update::IndexDocumentsMethod; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use uuid::Uuid; +use super::v4::{Task, TaskContent, TaskEvent}; use crate::index::{Settings, Unchecked}; -use crate::index_resolver::IndexUid; -use crate::tasks::task::{DocumentDeletion, Task, TaskContent, TaskEvent, TaskId, TaskResult}; +use crate::tasks::task::{DocumentDeletion, TaskId, TaskResult}; use super::v2; @@ -58,9 +59,9 @@ pub enum Update { ClearDocuments, } -impl From for TaskContent { - fn from(other: Update) -> Self { - match other { +impl From for super::v4::TaskContent { + fn from(update: Update) -> Self { + match update { Update::DeleteDocuments(ids) => { TaskContent::DocumentDeletion(DocumentDeletion::Ids(ids)) } @@ -185,10 +186,10 @@ impl Failed { impl From<(UpdateStatus, String, TaskId)> for Task { fn from((update, uid, task_id): (UpdateStatus, String, TaskId)) -> Self { // Dummy task - let mut task = Task { + let mut task = super::v4::Task { id: task_id, - index_uid: IndexUid::new(uid).unwrap(), - content: TaskContent::IndexDeletion, + index_uid: IndexUid::new_unchecked(uid), + content: super::v4::TaskContent::IndexDeletion, events: Vec::new(), }; diff --git a/meilisearch-lib/src/dump/compat/v4.rs b/meilisearch-lib/src/dump/compat/v4.rs new file mode 100644 index 000000000..c412e7f17 --- /dev/null +++ b/meilisearch-lib/src/dump/compat/v4.rs @@ -0,0 +1,145 @@ +use meilisearch_types::error::ResponseError; +use meilisearch_types::index_uid::IndexUid; +use milli::update::IndexDocumentsMethod; +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::index::{Settings, Unchecked}; +use crate::tasks::batch::BatchId; +use crate::tasks::task::{ + DocumentDeletion, TaskContent as NewTaskContent, TaskEvent as NewTaskEvent, TaskId, TaskResult, +}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Task { + pub id: TaskId, + pub index_uid: IndexUid, + pub content: TaskContent, + pub events: Vec, +} + +impl From for crate::tasks::task::Task { + fn from(other: Task) -> Self { + Self { + id: other.id, + content: NewTaskContent::from((other.index_uid, other.content)), + events: other.events.into_iter().map(Into::into).collect(), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum TaskEvent { + Created(#[serde(with = "time::serde::rfc3339")] OffsetDateTime), + Batched { + #[serde(with = "time::serde::rfc3339")] + timestamp: OffsetDateTime, + batch_id: BatchId, + }, + Processing(#[serde(with = "time::serde::rfc3339")] OffsetDateTime), + Succeded { + result: TaskResult, + #[serde(with = "time::serde::rfc3339")] + timestamp: OffsetDateTime, + }, + Failed { + error: ResponseError, + #[serde(with = "time::serde::rfc3339")] + timestamp: OffsetDateTime, + }, +} + +impl From for NewTaskEvent { + fn from(other: TaskEvent) -> Self { + match other { + TaskEvent::Created(x) => NewTaskEvent::Created(x), + TaskEvent::Batched { + timestamp, + batch_id, + } => NewTaskEvent::Batched { + timestamp, + batch_id, + }, + TaskEvent::Processing(x) => NewTaskEvent::Processing(x), + TaskEvent::Succeded { result, timestamp } => { + NewTaskEvent::Succeeded { result, timestamp } + } + TaskEvent::Failed { error, timestamp } => NewTaskEvent::Failed { error, timestamp }, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[allow(clippy::large_enum_variant)] +pub enum TaskContent { + DocumentAddition { + content_uuid: Uuid, + merge_strategy: IndexDocumentsMethod, + primary_key: Option, + documents_count: usize, + allow_index_creation: bool, + }, + DocumentDeletion(DocumentDeletion), + SettingsUpdate { + settings: Settings, + /// Indicates whether the task was a deletion + is_deletion: bool, + allow_index_creation: bool, + }, + IndexDeletion, + IndexCreation { + primary_key: Option, + }, + IndexUpdate { + primary_key: Option, + }, + Dump { + uid: String, + }, +} + +impl From<(IndexUid, TaskContent)> for NewTaskContent { + fn from((index_uid, content): (IndexUid, TaskContent)) -> Self { + match content { + TaskContent::DocumentAddition { + content_uuid, + merge_strategy, + primary_key, + documents_count, + allow_index_creation, + } => NewTaskContent::DocumentAddition { + index_uid, + content_uuid, + merge_strategy, + primary_key, + documents_count, + allow_index_creation, + }, + TaskContent::DocumentDeletion(deletion) => NewTaskContent::DocumentDeletion { + index_uid, + deletion, + }, + TaskContent::SettingsUpdate { + settings, + is_deletion, + allow_index_creation, + } => NewTaskContent::SettingsUpdate { + index_uid, + settings, + is_deletion, + allow_index_creation, + }, + TaskContent::IndexDeletion => NewTaskContent::IndexDeletion { index_uid }, + TaskContent::IndexCreation { primary_key } => NewTaskContent::IndexCreation { + index_uid, + primary_key, + }, + TaskContent::IndexUpdate { primary_key } => NewTaskContent::IndexUpdate { + index_uid, + primary_key, + }, + TaskContent::Dump { uid } => NewTaskContent::Dump { uid }, + } + } +} diff --git a/meilisearch-lib/src/dump/error.rs b/meilisearch-lib/src/dump/error.rs new file mode 100644 index 000000000..3f6e2aae5 --- /dev/null +++ b/meilisearch-lib/src/dump/error.rs @@ -0,0 +1,36 @@ +use meilisearch_auth::error::AuthControllerError; +use meilisearch_types::error::{Code, ErrorCode}; +use meilisearch_types::internal_error; + +use crate::{index_resolver::error::IndexResolverError, tasks::error::TaskError}; + +pub type Result = std::result::Result; + +#[derive(thiserror::Error, Debug)] +pub enum DumpError { + #[error("An internal error has occurred. `{0}`.")] + Internal(Box), + #[error("{0}")] + IndexResolver(#[from] IndexResolverError), +} + +internal_error!( + DumpError: milli::heed::Error, + std::io::Error, + tokio::task::JoinError, + tokio::sync::oneshot::error::RecvError, + serde_json::error::Error, + tempfile::PersistError, + fs_extra::error::Error, + AuthControllerError, + TaskError +); + +impl ErrorCode for DumpError { + fn error_code(&self) -> Code { + match self { + DumpError::Internal(_) => Code::Internal, + DumpError::IndexResolver(e) => e.error_code(), + } + } +} diff --git a/meilisearch-lib/src/dump/handler.rs b/meilisearch-lib/src/dump/handler.rs new file mode 100644 index 000000000..069196451 --- /dev/null +++ b/meilisearch-lib/src/dump/handler.rs @@ -0,0 +1,188 @@ +#[cfg(not(test))] +pub use real::DumpHandler; + +#[cfg(test)] +pub use test::MockDumpHandler as DumpHandler; + +use time::{macros::format_description, OffsetDateTime}; + +/// Generate uid from creation date +pub fn generate_uid() -> String { + OffsetDateTime::now_utc() + .format(format_description!( + "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" + )) + .unwrap() +} + +mod real { + use std::path::PathBuf; + use std::sync::Arc; + + use log::{info, trace}; + use meilisearch_auth::AuthController; + use milli::heed::Env; + use tokio::fs::create_dir_all; + use tokio::io::AsyncWriteExt; + + use crate::analytics; + use crate::compression::to_tar_gz; + use crate::dump::error::{DumpError, Result}; + use crate::dump::{MetadataVersion, META_FILE_NAME}; + use crate::index_resolver::{ + index_store::IndexStore, meta_store::IndexMetaStore, IndexResolver, + }; + use crate::tasks::TaskStore; + use crate::update_file_store::UpdateFileStore; + + pub struct DumpHandler { + dump_path: PathBuf, + db_path: PathBuf, + update_file_store: UpdateFileStore, + task_store_size: usize, + index_db_size: usize, + env: Arc, + index_resolver: Arc>, + } + + impl DumpHandler + where + U: IndexMetaStore + Sync + Send + 'static, + I: IndexStore + Sync + Send + 'static, + { + pub fn new( + dump_path: PathBuf, + db_path: PathBuf, + update_file_store: UpdateFileStore, + task_store_size: usize, + index_db_size: usize, + env: Arc, + index_resolver: Arc>, + ) -> Self { + Self { + dump_path, + db_path, + update_file_store, + task_store_size, + index_db_size, + env, + index_resolver, + } + } + + pub async fn run(&self, uid: String) -> Result<()> { + trace!("Performing dump."); + + create_dir_all(&self.dump_path).await?; + + let temp_dump_dir = tokio::task::spawn_blocking(tempfile::TempDir::new).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let meta = MetadataVersion::new_v5(self.index_db_size, self.task_store_size); + let meta_path = temp_dump_path.join(META_FILE_NAME); + + let meta_bytes = serde_json::to_vec(&meta)?; + let mut meta_file = tokio::fs::File::create(&meta_path).await?; + meta_file.write_all(&meta_bytes).await?; + + analytics::copy_user_id(&self.db_path, &temp_dump_path); + + create_dir_all(&temp_dump_path.join("indexes")).await?; + + let db_path = self.db_path.clone(); + let temp_dump_path_clone = temp_dump_path.clone(); + tokio::task::spawn_blocking(move || -> Result<()> { + AuthController::dump(db_path, temp_dump_path_clone)?; + Ok(()) + }) + .await??; + TaskStore::dump( + self.env.clone(), + &temp_dump_path, + self.update_file_store.clone(), + ) + .await?; + self.index_resolver.dump(&temp_dump_path).await?; + + let dump_path = self.dump_path.clone(); + let dump_path = tokio::task::spawn_blocking(move || -> Result { + // for now we simply copy the updates/updates_files + // FIXME: We may copy more files than necessary, if new files are added while we are + // performing the dump. We need a way to filter them out. + + let temp_dump_file = tempfile::NamedTempFile::new_in(&dump_path)?; + to_tar_gz(temp_dump_path, temp_dump_file.path()) + .map_err(|e| DumpError::Internal(e.into()))?; + + let dump_path = dump_path.join(uid).with_extension("dump"); + temp_dump_file.persist(&dump_path)?; + + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(()) + } + } +} + +#[cfg(test)] +mod test { + use std::path::PathBuf; + use std::sync::Arc; + + use milli::heed::Env; + use nelson::Mocker; + + use crate::dump::error::Result; + use crate::index_resolver::IndexResolver; + use crate::index_resolver::{index_store::IndexStore, meta_store::IndexMetaStore}; + use crate::update_file_store::UpdateFileStore; + + use super::*; + + pub enum MockDumpHandler { + Real(super::real::DumpHandler), + Mock(Mocker), + } + + impl MockDumpHandler { + pub fn mock(mocker: Mocker) -> Self { + Self::Mock(mocker) + } + } + + impl MockDumpHandler + where + U: IndexMetaStore + Sync + Send + 'static, + I: IndexStore + Sync + Send + 'static, + { + pub fn new( + dump_path: PathBuf, + db_path: PathBuf, + update_file_store: UpdateFileStore, + task_store_size: usize, + index_db_size: usize, + env: Arc, + index_resolver: Arc>, + ) -> Self { + Self::Real(super::real::DumpHandler::new( + dump_path, + db_path, + update_file_store, + task_store_size, + index_db_size, + env, + index_resolver, + )) + } + pub async fn run(&self, uid: String) -> Result<()> { + match self { + DumpHandler::Real(real) => real.run(uid).await, + DumpHandler::Mock(mocker) => unsafe { mocker.get("run").call(uid) }, + } + } + } +} diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/mod.rs b/meilisearch-lib/src/dump/loaders/mod.rs similarity index 75% rename from meilisearch-lib/src/index_controller/dump_actor/loaders/mod.rs rename to meilisearch-lib/src/dump/loaders/mod.rs index ecc305652..199b20c02 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/loaders/mod.rs +++ b/meilisearch-lib/src/dump/loaders/mod.rs @@ -1,3 +1,4 @@ pub mod v2; pub mod v3; pub mod v4; +pub mod v5; diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-lib/src/dump/loaders/v1.rs similarity index 100% rename from meilisearch-lib/src/index_controller/dump_actor/loaders/v1.rs rename to meilisearch-lib/src/dump/loaders/v1.rs diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-lib/src/dump/loaders/v2.rs similarity index 98% rename from meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs rename to meilisearch-lib/src/dump/loaders/v2.rs index e2445913e..5926de931 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-lib/src/dump/loaders/v2.rs @@ -5,8 +5,8 @@ use std::path::{Path, PathBuf}; use serde_json::{Deserializer, Value}; use tempfile::NamedTempFile; -use crate::index_controller::dump_actor::compat::{self, v2, v3}; -use crate::index_controller::dump_actor::Metadata; +use crate::dump::compat::{self, v2, v3}; +use crate::dump::Metadata; use crate::options::IndexerOpts; /// The dump v2 reads the dump folder and patches all the needed file to make it compatible with a diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/v3.rs b/meilisearch-lib/src/dump/loaders/v3.rs similarity index 94% rename from meilisearch-lib/src/index_controller/dump_actor/loaders/v3.rs rename to meilisearch-lib/src/dump/loaders/v3.rs index 902691511..44984c946 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/loaders/v3.rs +++ b/meilisearch-lib/src/dump/loaders/v3.rs @@ -9,11 +9,11 @@ use log::info; use tempfile::tempdir; use uuid::Uuid; -use crate::index_controller::dump_actor::compat::v3; -use crate::index_controller::dump_actor::Metadata; +use crate::dump::compat::{self, v3}; +use crate::dump::Metadata; use crate::index_resolver::meta_store::{DumpEntry, IndexMeta}; use crate::options::IndexerOpts; -use crate::tasks::task::{Task, TaskId}; +use crate::tasks::task::TaskId; /// dump structure for V3: /// . @@ -124,7 +124,7 @@ fn patch_updates( .clone(); serde_json::to_writer( &mut dst_file, - &Task::from((entry.update, name, task_id as TaskId)), + &compat::v4::Task::from((entry.update, name, task_id as TaskId)), )?; dst_file.write_all(b"\n")?; Ok(()) diff --git a/meilisearch-lib/src/dump/loaders/v4.rs b/meilisearch-lib/src/dump/loaders/v4.rs new file mode 100644 index 000000000..0744df7ea --- /dev/null +++ b/meilisearch-lib/src/dump/loaders/v4.rs @@ -0,0 +1,103 @@ +use std::fs::{self, create_dir_all, File}; +use std::io::{BufReader, Write}; +use std::path::Path; + +use fs_extra::dir::{self, CopyOptions}; +use log::info; +use serde_json::{Deserializer, Map, Value}; +use tempfile::tempdir; +use uuid::Uuid; + +use crate::dump::{compat, Metadata}; +use crate::options::IndexerOpts; +use crate::tasks::task::Task; + +pub fn load_dump( + meta: Metadata, + src: impl AsRef, + dst: impl AsRef, + index_db_size: usize, + meta_env_size: usize, + indexing_options: &IndexerOpts, +) -> anyhow::Result<()> { + info!("Patching dump V4 to dump V5..."); + + let patched_dir = tempdir()?; + let options = CopyOptions::default(); + + // Indexes + dir::copy(src.as_ref().join("indexes"), &patched_dir, &options)?; + + // Index uuids + dir::copy(src.as_ref().join("index_uuids"), &patched_dir, &options)?; + + // Metadata + fs::copy( + src.as_ref().join("metadata.json"), + patched_dir.path().join("metadata.json"), + )?; + + // Updates + patch_updates(&src, &patched_dir)?; + + // Keys + patch_keys(&src, &patched_dir)?; + + super::v5::load_dump( + meta, + &patched_dir, + dst, + index_db_size, + meta_env_size, + indexing_options, + ) +} + +fn patch_updates(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + let updates_path = src.as_ref().join("updates/data.jsonl"); + let output_updates_path = dst.as_ref().join("updates/data.jsonl"); + create_dir_all(output_updates_path.parent().unwrap())?; + let udpates_file = File::open(updates_path)?; + let mut output_update_file = File::create(output_updates_path)?; + + serde_json::Deserializer::from_reader(udpates_file) + .into_iter::() + .try_for_each(|task| -> anyhow::Result<()> { + let task: Task = task?.into(); + + serde_json::to_writer(&mut output_update_file, &task)?; + output_update_file.write_all(b"\n")?; + + Ok(()) + })?; + + output_update_file.flush()?; + + Ok(()) +} + +fn patch_keys(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + let keys_file_src = src.as_ref().join("keys"); + + if !keys_file_src.exists() { + return Ok(()); + } + + fs::create_dir_all(&dst)?; + let keys_file_dst = dst.as_ref().join("keys"); + let mut writer = File::create(&keys_file_dst)?; + + let reader = BufReader::new(File::open(&keys_file_src)?); + for key in Deserializer::from_reader(reader).into_iter() { + let mut key: Map = key?; + + // generate a new uuid v4 and insert it in the key. + let uid = serde_json::to_value(Uuid::new_v4()).unwrap(); + key.insert("uid".to_string(), uid); + + serde_json::to_writer(&mut writer, &key)?; + writer.write_all(b"\n")?; + } + + Ok(()) +} diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/v4.rs b/meilisearch-lib/src/dump/loaders/v5.rs similarity index 91% rename from meilisearch-lib/src/index_controller/dump_actor/loaders/v4.rs rename to meilisearch-lib/src/dump/loaders/v5.rs index 38d61f146..fcb4224bb 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/loaders/v4.rs +++ b/meilisearch-lib/src/dump/loaders/v5.rs @@ -1,12 +1,11 @@ -use std::path::Path; -use std::sync::Arc; +use std::{path::Path, sync::Arc}; use log::info; use meilisearch_auth::AuthController; use milli::heed::EnvOpenOptions; use crate::analytics; -use crate::index_controller::dump_actor::Metadata; +use crate::dump::Metadata; use crate::index_resolver::IndexResolver; use crate::options::IndexerOpts; use crate::tasks::TaskStore; @@ -21,7 +20,7 @@ pub fn load_dump( indexing_options: &IndexerOpts, ) -> anyhow::Result<()> { info!( - "Loading dump from {}, dump database version: {}, dump version: V4", + "Loading dump from {}, dump database version: {}, dump version: V5", meta.dump_date, meta.db_version ); diff --git a/meilisearch-lib/src/dump/mod.rs b/meilisearch-lib/src/dump/mod.rs new file mode 100644 index 000000000..ea7b9f3dc --- /dev/null +++ b/meilisearch-lib/src/dump/mod.rs @@ -0,0 +1,262 @@ +use std::fs::File; +use std::path::Path; + +use anyhow::bail; +use log::info; +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; + +use tempfile::TempDir; + +use crate::compression::from_tar_gz; +use crate::options::IndexerOpts; + +use self::loaders::{v2, v3, v4, v5}; + +pub use handler::{generate_uid, DumpHandler}; + +mod compat; +pub mod error; +mod handler; +mod loaders; + +const META_FILE_NAME: &str = "metadata.json"; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Metadata { + db_version: String, + index_db_size: usize, + update_db_size: usize, + #[serde(with = "time::serde::rfc3339")] + dump_date: OffsetDateTime, +} + +impl Metadata { + pub fn new(index_db_size: usize, update_db_size: usize) -> Self { + Self { + db_version: env!("CARGO_PKG_VERSION").to_string(), + index_db_size, + update_db_size, + dump_date: OffsetDateTime::now_utc(), + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct MetadataV1 { + pub db_version: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "dumpVersion")] +pub enum MetadataVersion { + V1(MetadataV1), + V2(Metadata), + V3(Metadata), + V4(Metadata), + // V5 is forward compatible with V4 but not backward compatible. + V5(Metadata), +} + +impl MetadataVersion { + pub fn load_dump( + self, + src: impl AsRef, + dst: impl AsRef, + index_db_size: usize, + meta_env_size: usize, + indexing_options: &IndexerOpts, + ) -> anyhow::Result<()> { + match self { + MetadataVersion::V1(_meta) => { + anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.") + } + MetadataVersion::V2(meta) => v2::load_dump( + meta, + src, + dst, + index_db_size, + meta_env_size, + indexing_options, + )?, + MetadataVersion::V3(meta) => v3::load_dump( + meta, + src, + dst, + index_db_size, + meta_env_size, + indexing_options, + )?, + MetadataVersion::V4(meta) => v4::load_dump( + meta, + src, + dst, + index_db_size, + meta_env_size, + indexing_options, + )?, + MetadataVersion::V5(meta) => v5::load_dump( + meta, + src, + dst, + index_db_size, + meta_env_size, + indexing_options, + )?, + } + + Ok(()) + } + + pub fn new_v5(index_db_size: usize, update_db_size: usize) -> Self { + let meta = Metadata::new(index_db_size, update_db_size); + Self::V5(meta) + } + + pub fn db_version(&self) -> &str { + match self { + Self::V1(meta) => &meta.db_version, + Self::V2(meta) | Self::V3(meta) | Self::V4(meta) | Self::V5(meta) => &meta.db_version, + } + } + + pub fn version(&self) -> &'static str { + match self { + MetadataVersion::V1(_) => "V1", + MetadataVersion::V2(_) => "V2", + MetadataVersion::V3(_) => "V3", + MetadataVersion::V4(_) => "V4", + MetadataVersion::V5(_) => "V5", + } + } + + pub fn dump_date(&self) -> Option<&OffsetDateTime> { + match self { + MetadataVersion::V1(_) => None, + MetadataVersion::V2(meta) + | MetadataVersion::V3(meta) + | MetadataVersion::V4(meta) + | MetadataVersion::V5(meta) => Some(&meta.dump_date), + } + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] +#[serde(rename_all = "snake_case")] +pub enum DumpStatus { + Done, + InProgress, + Failed, +} + +pub fn load_dump( + dst_path: impl AsRef, + src_path: impl AsRef, + ignore_dump_if_db_exists: bool, + ignore_missing_dump: bool, + index_db_size: usize, + update_db_size: usize, + indexer_opts: &IndexerOpts, +) -> anyhow::Result<()> { + let empty_db = crate::is_empty_db(&dst_path); + let src_path_exists = src_path.as_ref().exists(); + + if empty_db && src_path_exists { + let (tmp_src, tmp_dst, meta) = extract_dump(&dst_path, &src_path)?; + meta.load_dump( + tmp_src.path(), + tmp_dst.path(), + index_db_size, + update_db_size, + indexer_opts, + )?; + persist_dump(&dst_path, tmp_dst)?; + Ok(()) + } else if !empty_db && !ignore_dump_if_db_exists { + bail!( + "database already exists at {:?}, try to delete it or rename it", + dst_path + .as_ref() + .canonicalize() + .unwrap_or_else(|_| dst_path.as_ref().to_owned()) + ) + } else if !src_path_exists && !ignore_missing_dump { + bail!("dump doesn't exist at {:?}", src_path.as_ref()) + } else { + // there is nothing to do + Ok(()) + } +} + +fn extract_dump( + dst_path: impl AsRef, + src_path: impl AsRef, +) -> anyhow::Result<(TempDir, TempDir, MetadataVersion)> { + // Setup a temp directory path in the same path as the database, to prevent cross devices + // references. + let temp_path = dst_path + .as_ref() + .parent() + .map(ToOwned::to_owned) + .unwrap_or_else(|| ".".into()); + + let tmp_src = tempfile::tempdir_in(temp_path)?; + let tmp_src_path = tmp_src.path(); + + from_tar_gz(&src_path, tmp_src_path)?; + + let meta_path = tmp_src_path.join(META_FILE_NAME); + let mut meta_file = File::open(&meta_path)?; + let meta: MetadataVersion = serde_json::from_reader(&mut meta_file)?; + + if !dst_path.as_ref().exists() { + std::fs::create_dir_all(dst_path.as_ref())?; + } + + let tmp_dst = tempfile::tempdir_in(dst_path.as_ref())?; + + info!( + "Loading dump {}, dump database version: {}, dump version: {}", + meta.dump_date() + .map(|t| format!("from {}", t)) + .unwrap_or_else(String::new), + meta.db_version(), + meta.version() + ); + + Ok((tmp_src, tmp_dst, meta)) +} + +fn persist_dump(dst_path: impl AsRef, tmp_dst: TempDir) -> anyhow::Result<()> { + let persisted_dump = tmp_dst.into_path(); + + // Delete everything in the `data.ms` except the tempdir. + if dst_path.as_ref().exists() { + for file in dst_path.as_ref().read_dir().unwrap() { + let file = file.unwrap().path(); + if file.file_name() == persisted_dump.file_name() { + continue; + } + + if file.is_file() { + std::fs::remove_file(&file)?; + } else { + std::fs::remove_dir_all(&file)?; + } + } + } + + // Move the whole content of the tempdir into the `data.ms`. + for file in persisted_dump.read_dir().unwrap() { + let file = file.unwrap().path(); + + std::fs::rename(&file, &dst_path.as_ref().join(file.file_name().unwrap()))?; + } + + // Delete the empty tempdir. + std::fs::remove_dir_all(&persisted_dump)?; + + Ok(()) +} diff --git a/meilisearch-lib/src/error.rs b/meilisearch-lib/src/error.rs index c3e7b8313..83e9263b4 100644 --- a/meilisearch-lib/src/error.rs +++ b/meilisearch-lib/src/error.rs @@ -1,7 +1,7 @@ use std::error::Error; use std::fmt; -use meilisearch_error::{Code, ErrorCode}; +use meilisearch_types::error::{Code, ErrorCode}; use milli::UserError; #[derive(Debug)] diff --git a/meilisearch-lib/src/index/dump.rs b/meilisearch-lib/src/index/dump.rs index e201e738b..c6feb187f 100644 --- a/meilisearch-lib/src/index/dump.rs +++ b/meilisearch-lib/src/index/dump.rs @@ -27,7 +27,7 @@ const DATA_FILE_NAME: &str = "documents.jsonl"; impl Index { pub fn dump(&self, path: impl AsRef) -> Result<()> { // acquire write txn make sure any ongoing write is finished before we start. - let txn = self.env.write_txn()?; + let txn = self.write_txn()?; let path = path.as_ref().join(format!("indexes/{}", self.uuid)); create_dir_all(&path)?; diff --git a/meilisearch-lib/src/index/error.rs b/meilisearch-lib/src/index/error.rs index 89a12a41f..e31fcc4a0 100644 --- a/meilisearch-lib/src/index/error.rs +++ b/meilisearch-lib/src/index/error.rs @@ -1,6 +1,7 @@ use std::error::Error; -use meilisearch_error::{internal_error, Code, ErrorCode}; +use meilisearch_types::error::{Code, ErrorCode}; +use meilisearch_types::internal_error; use serde_json::Value; use crate::{error::MilliError, update_file_store}; diff --git a/meilisearch-lib/src/index/index.rs b/meilisearch-lib/src/index/index.rs index f5122c8c1..518e9ce3e 100644 --- a/meilisearch-lib/src/index/index.rs +++ b/meilisearch-lib/src/index/index.rs @@ -1,24 +1,25 @@ -use std::collections::{BTreeSet, HashSet}; +use std::collections::BTreeSet; use std::fs::create_dir_all; use std::marker::PhantomData; use std::ops::Deref; use std::path::Path; use std::sync::Arc; +use walkdir::WalkDir; use fst::IntoStreamer; -use milli::heed::{EnvOpenOptions, RoTxn}; +use milli::heed::{CompactionOption, EnvOpenOptions, RoTxn}; use milli::update::{IndexerConfig, Setting}; -use milli::{obkv_to_json, FieldDistribution, FieldId}; +use milli::{obkv_to_json, FieldDistribution, DEFAULT_VALUES_PER_FACET}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use time::OffsetDateTime; use uuid::Uuid; -use crate::EnvSizer; +use crate::index::search::DEFAULT_PAGINATION_MAX_TOTAL_HITS; use super::error::IndexError; use super::error::Result; -use super::updates::{MinWordSizeTyposSetting, TypoSettings}; +use super::updates::{FacetingSettings, MinWordSizeTyposSetting, PaginationSettings, TypoSettings}; use super::{Checked, Settings}; pub type Document = Map; @@ -175,12 +176,10 @@ impl Index { two_typos: Setting::Set(self.min_word_len_two_typos(txn)?), }; - let disabled_words = self - .exact_words(txn)? - .into_stream() - .into_strs()? - .into_iter() - .collect(); + let disabled_words = match self.exact_words(txn)? { + Some(fst) => fst.into_stream().into_strs()?.into_iter().collect(), + None => BTreeSet::new(), + }; let disabled_attributes = self .exact_attributes(txn)? @@ -195,6 +194,20 @@ impl Index { disable_on_attributes: Setting::Set(disabled_attributes), }; + let faceting = FacetingSettings { + max_values_per_facet: Setting::Set( + self.max_values_per_facet(txn)? + .unwrap_or(DEFAULT_VALUES_PER_FACET), + ), + }; + + let pagination = PaginationSettings { + max_total_hits: Setting::Set( + self.pagination_max_total_hits(txn)? + .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS), + ), + }; + Ok(Settings { displayed_attributes: match displayed_attributes { Some(attrs) => Setting::Set(attrs), @@ -214,46 +227,55 @@ impl Index { }, synonyms: Setting::Set(synonyms), typo_tolerance: Setting::Set(typo_tolerance), + faceting: Setting::Set(faceting), + pagination: Setting::Set(pagination), _kind: PhantomData, }) } + /// Return the total number of documents contained in the index + the selected documents. pub fn retrieve_documents>( &self, offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> Result>> { + ) -> Result<(u64, Vec)> { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - let fields_to_display = - self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; + let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); - let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit); + let iter = self.all_documents(&txn)?.skip(offset).take(limit); let mut documents = Vec::new(); for entry in iter { let (_id, obkv) = entry?; - let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?; - documents.push(object); + let document = obkv_to_json(&all_fields, &fields_ids_map, obkv)?; + let document = match &attributes_to_retrieve { + Some(attributes_to_retrieve) => permissive_json_pointer::select_values( + &document, + attributes_to_retrieve.iter().map(|s| s.as_ref()), + ), + None => document, + }; + documents.push(document); } - Ok(documents) + let number_of_documents = self.number_of_documents(&txn)?; + + Ok((number_of_documents, documents)) } pub fn retrieve_document>( &self, doc_id: String, attributes_to_retrieve: Option>, - ) -> Result> { + ) -> Result { let txn = self.read_txn()?; let fields_ids_map = self.fields_ids_map(&txn)?; - - let fields_to_display = - self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?; + let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); let internal_id = self .external_documents_ids(&txn)? @@ -267,36 +289,25 @@ impl Index { .map(|(_, d)| d) .ok_or(IndexError::DocumentNotFound(doc_id))?; - let document = obkv_to_json(&fields_to_display, &fields_ids_map, document)?; + let document = obkv_to_json(&all_fields, &fields_ids_map, document)?; + let document = match &attributes_to_retrieve { + Some(attributes_to_retrieve) => permissive_json_pointer::select_values( + &document, + attributes_to_retrieve.iter().map(|s| s.as_ref()), + ), + None => document, + }; Ok(document) } pub fn size(&self) -> u64 { - self.env.size() - } - - fn fields_to_display>( - &self, - txn: &milli::heed::RoTxn, - attributes_to_retrieve: &Option>, - fields_ids_map: &milli::FieldsIdsMap, - ) -> Result> { - let mut displayed_fields_ids = match self.displayed_fields_ids(txn)? { - Some(ids) => ids.into_iter().collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - let attributes_to_retrieve_ids = match attributes_to_retrieve { - Some(attrs) => attrs - .iter() - .filter_map(|f| fields_ids_map.id(f.as_ref())) - .collect::>(), - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid)); - Ok(displayed_fields_ids) + WalkDir::new(self.inner.path()) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len()) } pub fn snapshot(&self, path: impl AsRef) -> Result<()> { @@ -304,9 +315,7 @@ impl Index { create_dir_all(&dst)?; dst.push("data.mdb"); let _txn = self.write_txn()?; - self.inner - .env - .copy_to_path(dst, milli::heed::CompactionOption::Enabled)?; + self.inner.copy_to_path(dst, CompactionOption::Enabled)?; Ok(()) } } diff --git a/meilisearch-lib/src/index/mod.rs b/meilisearch-lib/src/index/mod.rs index 3a42b2617..e6c831a01 100644 --- a/meilisearch-lib/src/index/mod.rs +++ b/meilisearch-lib/src/index/mod.rs @@ -1,6 +1,5 @@ pub use search::{ - default_crop_length, default_crop_marker, default_highlight_post_tag, - default_highlight_pre_tag, SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, + SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, }; pub use updates::{apply_settings_to_builder, Checked, Facets, Settings, Unchecked}; @@ -32,11 +31,11 @@ pub mod test { use milli::update::IndexerConfig; use milli::update::{DocumentAdditionResult, DocumentDeletionResult, IndexDocumentsMethod}; use nelson::Mocker; - use serde_json::{Map, Value}; use uuid::Uuid; use super::error::Result; use super::index::Index; + use super::Document; use super::{Checked, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings}; use crate::update_file_store::UpdateFileStore; @@ -102,7 +101,7 @@ pub mod test { offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> Result>> { + ) -> Result<(u64, Vec)> { match self { MockIndex::Real(index) => { index.retrieve_documents(offset, limit, attributes_to_retrieve) @@ -115,7 +114,7 @@ pub mod test { &self, doc_id: String, attributes_to_retrieve: Option>, - ) -> Result> { + ) -> Result { match self { MockIndex::Real(index) => index.retrieve_document(doc_id, attributes_to_retrieve), MockIndex::Mock(_) => todo!(), diff --git a/meilisearch-lib/src/index/search.rs b/meilisearch-lib/src/index/search.rs index 7c12f985e..58bcf7ef4 100644 --- a/meilisearch-lib/src/index/search.rs +++ b/meilisearch-lib/src/index/search.rs @@ -4,8 +4,11 @@ use std::str::FromStr; use std::time::Instant; use either::Either; -use milli::tokenizer::{Analyzer, AnalyzerConfig, Token}; -use milli::{AscDesc, FieldId, FieldsIdsMap, Filter, MatchingWords, SortError}; +use milli::tokenizer::TokenizerBuilder; +use milli::{ + AscDesc, FieldId, FieldsIdsMap, Filter, FormatOptions, MatchBounds, MatcherBuilder, SortError, + DEFAULT_VALUES_PER_FACET, +}; use regex::Regex; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -16,66 +19,41 @@ use super::error::{IndexError, Result}; use super::index::Index; pub type Document = serde_json::Map; -type MatchesInfo = BTreeMap>; +type MatchesPosition = BTreeMap>; -#[derive(Serialize, Debug, Clone, PartialEq)] -pub struct MatchInfo { - start: usize, - length: usize, -} - -pub const DEFAULT_SEARCH_LIMIT: usize = 20; -const fn default_search_limit() -> usize { - DEFAULT_SEARCH_LIMIT -} - -pub const DEFAULT_CROP_LENGTH: usize = 10; -pub const fn default_crop_length() -> usize { - DEFAULT_CROP_LENGTH -} - -pub const DEFAULT_CROP_MARKER: &str = "…"; -pub fn default_crop_marker() -> String { - DEFAULT_CROP_MARKER.to_string() -} - -pub const DEFAULT_HIGHLIGHT_PRE_TAG: &str = ""; -pub fn default_highlight_pre_tag() -> String { - DEFAULT_HIGHLIGHT_PRE_TAG.to_string() -} - -pub const DEFAULT_HIGHLIGHT_POST_TAG: &str = ""; -pub fn default_highlight_post_tag() -> String { - DEFAULT_HIGHLIGHT_POST_TAG.to_string() -} +pub const DEFAULT_SEARCH_LIMIT: fn() -> usize = || 20; +pub const DEFAULT_CROP_LENGTH: fn() -> usize = || 10; +pub const DEFAULT_CROP_MARKER: fn() -> String = || "…".to_string(); +pub const DEFAULT_HIGHLIGHT_PRE_TAG: fn() -> String = || "".to_string(); +pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "".to_string(); /// The maximimum number of results that the engine /// will be able to return in one search call. -pub const HARD_RESULT_LIMIT: usize = 1000; +pub const DEFAULT_PAGINATION_MAX_TOTAL_HITS: usize = 1000; #[derive(Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchQuery { pub q: Option, pub offset: Option, - #[serde(default = "default_search_limit")] + #[serde(default = "DEFAULT_SEARCH_LIMIT")] pub limit: usize, pub attributes_to_retrieve: Option>, pub attributes_to_crop: Option>, - #[serde(default = "default_crop_length")] + #[serde(default = "DEFAULT_CROP_LENGTH")] pub crop_length: usize, pub attributes_to_highlight: Option>, // Default to false #[serde(default = "Default::default")] - pub matches: bool, + pub show_matches_position: bool, pub filter: Option, pub sort: Option>, - pub facets_distribution: Option>, - #[serde(default = "default_highlight_pre_tag")] + pub facets: Option>, + #[serde(default = "DEFAULT_HIGHLIGHT_PRE_TAG")] pub highlight_pre_tag: String, - #[serde(default = "default_highlight_post_tag")] + #[serde(default = "DEFAULT_HIGHLIGHT_POST_TAG")] pub highlight_post_tag: String, - #[serde(default = "default_crop_marker")] + #[serde(default = "DEFAULT_CROP_MARKER")] pub crop_marker: String, } @@ -85,39 +63,21 @@ pub struct SearchHit { pub document: Document, #[serde(rename = "_formatted", skip_serializing_if = "Document::is_empty")] pub formatted: Document, - #[serde(rename = "_matchesInfo", skip_serializing_if = "Option::is_none")] - pub matches_info: Option, + #[serde(rename = "_matchesPosition", skip_serializing_if = "Option::is_none")] + pub matches_position: Option, } #[derive(Serialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct SearchResult { pub hits: Vec, - pub nb_hits: u64, - pub exhaustive_nb_hits: bool, + pub estimated_total_hits: u64, pub query: String, pub limit: usize, pub offset: usize, pub processing_time_ms: u128, #[serde(skip_serializing_if = "Option::is_none")] - pub facets_distribution: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub exhaustive_facets_count: Option, -} - -#[derive(Copy, Clone, Default)] -struct FormatOptions { - highlight: bool, - crop: Option, -} - -impl FormatOptions { - pub fn merge(self, other: Self) -> Self { - Self { - highlight: self.highlight || other.highlight, - crop: self.crop.or(other.crop), - } - } + pub facet_distribution: Option>>, } impl Index { @@ -131,10 +91,14 @@ impl Index { search.query(query); } + let max_total_hits = self + .pagination_max_total_hits(&rtxn)? + .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS); + // Make sure that a user can't get more documents than the hard limit, // we align that on the offset too. - let offset = min(query.offset.unwrap_or(0), HARD_RESULT_LIMIT); - let limit = min(query.limit, HARD_RESULT_LIMIT.saturating_sub(offset)); + let offset = min(query.offset.unwrap_or(0), max_total_hits); + let limit = min(query.limit, max_total_hits.saturating_sub(offset)); search.offset(offset); search.limit(limit); @@ -216,16 +180,12 @@ impl Index { &displayed_ids, ); - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); + let tokenizer = TokenizerBuilder::default().build(); - let formatter = Formatter::new( - &analyzer, - (query.highlight_pre_tag, query.highlight_post_tag), - query.crop_marker, - ); + let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer); + formatter_builder.crop_marker(query.crop_marker); + formatter_builder.highlight_prefix(query.highlight_pre_tag); + formatter_builder.highlight_suffix(query.highlight_post_tag); let mut documents = Vec::new(); @@ -242,16 +202,13 @@ impl Index { let mut document = permissive_json_pointer::select_values(&displayed_document, attributes_to_retrieve); - let matches_info = query - .matches - .then(|| compute_matches(&matching_words, &document, &analyzer)); - - let formatted = format_fields( + let (matches_position, formatted) = format_fields( &displayed_document, &fields_ids_map, - &formatter, - &matching_words, + &formatter_builder, &formatted_options, + query.show_matches_position, + &displayed_ids, )?; if let Some(sort) = query.sort.as_ref() { @@ -261,38 +218,40 @@ impl Index { let hit = SearchHit { document, formatted, - matches_info, + matches_position, }; documents.push(hit); } - let nb_hits = candidates.len(); + let estimated_total_hits = candidates.len(); - let facets_distribution = match query.facets_distribution { + let facet_distribution = match query.facets { Some(ref fields) => { - let mut facets_distribution = self.facets_distribution(&rtxn); + let mut facet_distribution = self.facets_distribution(&rtxn); + + let max_values_by_facet = self + .max_values_per_facet(&rtxn)? + .unwrap_or(DEFAULT_VALUES_PER_FACET); + facet_distribution.max_values_per_facet(max_values_by_facet); + if fields.iter().all(|f| f != "*") { - facets_distribution.facets(fields); + facet_distribution.facets(fields); } - let distribution = facets_distribution.candidates(candidates).execute()?; + let distribution = facet_distribution.candidates(candidates).execute()?; Some(distribution) } None => None, }; - let exhaustive_facets_count = facets_distribution.as_ref().map(|_| false); // not implemented yet - let result = SearchResult { - exhaustive_nb_hits: false, // not implemented yet hits: documents, - nb_hits, + estimated_total_hits, query: query.q.clone().unwrap_or_default(), limit: query.limit, offset: query.offset.unwrap_or_default(), processing_time_ms: before_search.elapsed().as_millis(), - facets_distribution, - exhaustive_facets_count, + facet_distribution, }; Ok(result) } @@ -317,56 +276,6 @@ fn insert_geo_distance(sorts: &[String], document: &mut Document) { } } -fn compute_matches>( - matcher: &impl Matcher, - document: &Document, - analyzer: &Analyzer, -) -> MatchesInfo { - let mut matches = BTreeMap::new(); - - for (key, value) in document { - let mut infos = Vec::new(); - compute_value_matches(&mut infos, value, matcher, analyzer); - if !infos.is_empty() { - matches.insert(key.clone(), infos); - } - } - matches -} - -fn compute_value_matches<'a, A: AsRef<[u8]>>( - infos: &mut Vec, - value: &Value, - matcher: &impl Matcher, - analyzer: &Analyzer<'a, A>, -) { - match value { - Value::String(s) => { - let analyzed = analyzer.analyze(s); - let mut start = 0; - for (word, token) in analyzed.reconstruct() { - if token.is_word() { - if let Some(length) = matcher.matches(&token) { - infos.push(MatchInfo { start, length }); - } - } - - start += word.len(); - } - } - Value::Array(vals) => vals - .iter() - .for_each(|val| compute_value_matches(infos, val, matcher, analyzer)), - Value::Object(vals) => vals - .values() - .for_each(|val| compute_value_matches(infos, val, matcher, analyzer)), - Value::Number(number) => { - compute_value_matches(infos, &Value::String(number.to_string()), matcher, analyzer) - } - _ => (), - } -} - fn compute_formatted_options( attr_to_highlight: &HashSet, attr_to_crop: &[String], @@ -509,22 +418,22 @@ fn make_document( Ok(document) } -fn format_fields>( +fn format_fields<'a, A: AsRef<[u8]>>( document: &Document, field_ids_map: &FieldsIdsMap, - formatter: &Formatter, - matching_words: &impl Matcher, + builder: &MatcherBuilder<'a, A>, formatted_options: &BTreeMap, -) -> Result { - let selectors: Vec<_> = formatted_options - .keys() - // This unwrap must be safe since we got the ids from the fields_ids_map just - // before. - .map(|&fid| field_ids_map.name(fid).unwrap()) - .collect(); - let mut document = permissive_json_pointer::select_values(document, selectors.iter().copied()); + compute_matches: bool, + displayable_ids: &BTreeSet, +) -> Result<(Option, Document)> { + let mut matches_position = compute_matches.then(BTreeMap::new); + let mut document = document.clone(); - permissive_json_pointer::map_leaf_values(&mut document, selectors, |key, value| { + // select the attributes to retrieve + let displayable_names = displayable_ids + .iter() + .map(|&fid| field_ids_map.name(fid).expect("Missing field name")); + permissive_json_pointer::map_leaf_values(&mut document, displayable_names, |key, value| { // To get the formatting option of each key we need to see all the rules that applies // to the value and merge them together. eg. If a user said he wanted to highlight `doggo` // and crop `doggo.name`. `doggo.name` needs to be highlighted + cropped while `doggo.age` is only @@ -535,235 +444,113 @@ fn format_fields>( let name = field_ids_map.name(**field).unwrap(); milli::is_faceted_by(name, key) || milli::is_faceted_by(key, name) }) - .fold(FormatOptions::default(), |acc, (_, option)| { - acc.merge(*option) - }); - *value = formatter.format_value(std::mem::take(value), matching_words, format); + .map(|(_, option)| *option) + .reduce(|acc, option| acc.merge(option)); + let mut infos = Vec::new(); + + *value = format_value( + std::mem::take(value), + builder, + format, + &mut infos, + compute_matches, + ); + + if let Some(matches) = matches_position.as_mut() { + if !infos.is_empty() { + matches.insert(key.to_owned(), infos); + } + } }); - Ok(document) + let selectors = formatted_options + .keys() + // This unwrap must be safe since we got the ids from the fields_ids_map just + // before. + .map(|&fid| field_ids_map.name(fid).unwrap()); + let document = permissive_json_pointer::select_values(&document, selectors); + + Ok((matches_position, document)) } -/// trait to allow unit testing of `format_fields` -trait Matcher { - fn matches(&self, w: &Token) -> Option; -} - -#[cfg(test)] -impl Matcher for BTreeMap<&str, Option> { - fn matches(&self, w: &Token) -> Option { - self.get(w.text()).cloned().flatten() - } -} - -impl Matcher for MatchingWords { - fn matches(&self, w: &Token) -> Option { - self.matching_bytes(w) - } -} - -struct Formatter<'a, A> { - analyzer: &'a Analyzer<'a, A>, - highlight_tags: (String, String), - crop_marker: String, -} - -impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { - pub fn new( - analyzer: &'a Analyzer<'a, A>, - highlight_tags: (String, String), - crop_marker: String, - ) -> Self { - Self { - analyzer, - highlight_tags, - crop_marker, - } - } - - fn format_value( - &self, - value: Value, - matcher: &impl Matcher, - format_options: FormatOptions, - ) -> Value { - match value { - Value::String(old_string) => { - let value = self.format_string(old_string, matcher, format_options); - Value::String(value) +fn format_value<'a, A: AsRef<[u8]>>( + value: Value, + builder: &MatcherBuilder<'a, A>, + format_options: Option, + infos: &mut Vec, + compute_matches: bool, +) -> Value { + match value { + Value::String(old_string) => { + let mut matcher = builder.build(&old_string); + if compute_matches { + let matches = matcher.matches(); + infos.extend_from_slice(&matches[..]); } - Value::Array(values) => Value::Array( - values - .into_iter() - .map(|v| { - self.format_value( + + match format_options { + Some(format_options) => { + let value = matcher.format(format_options); + Value::String(value.into_owned()) + } + None => Value::String(old_string), + } + } + Value::Array(values) => Value::Array( + values + .into_iter() + .map(|v| { + format_value( + v, + builder, + format_options.map(|format_options| FormatOptions { + highlight: format_options.highlight, + crop: None, + }), + infos, + compute_matches, + ) + }) + .collect(), + ), + Value::Object(object) => Value::Object( + object + .into_iter() + .map(|(k, v)| { + ( + k, + format_value( v, - matcher, - FormatOptions { + builder, + format_options.map(|format_options| FormatOptions { highlight: format_options.highlight, crop: None, - }, - ) - }) - .collect(), - ), - Value::Object(object) => Value::Object( - object - .into_iter() - .map(|(k, v)| { - ( - k, - self.format_value( - v, - matcher, - FormatOptions { - highlight: format_options.highlight, - crop: None, - }, - ), - ) - }) - .collect(), - ), - Value::Number(number) => { - let number_string_value = - self.format_string(number.to_string(), matcher, format_options); - Value::String(number_string_value) + }), + infos, + compute_matches, + ), + ) + }) + .collect(), + ), + Value::Number(number) => { + let s = number.to_string(); + + let mut matcher = builder.build(&s); + if compute_matches { + let matches = matcher.matches(); + infos.extend_from_slice(&matches[..]); + } + + match format_options { + Some(format_options) => { + let value = matcher.format(format_options); + Value::String(value.into_owned()) + } + None => Value::Number(number), } - value => value, } - } - - fn format_string( - &self, - s: String, - matcher: &impl Matcher, - format_options: FormatOptions, - ) -> String { - let analyzed = self.analyzer.analyze(&s); - - let mut tokens = analyzed.reconstruct(); - let mut crop_marker_before = false; - - let tokens_interval: Box> = match format_options.crop { - Some(crop_len) if crop_len > 0 => { - let mut buffer = Vec::new(); - let mut tokens = tokens.by_ref().peekable(); - - while let Some((word, token)) = - tokens.next_if(|(_, token)| matcher.matches(token).is_none()) - { - buffer.push((word, token)); - } - - match tokens.next() { - Some(token) => { - let mut total_count: usize = buffer - .iter() - .filter(|(_, token)| token.is_separator().is_none()) - .count(); - - let crop_len_before = crop_len / 2; - // check if start will be cropped. - crop_marker_before = total_count > crop_len_before; - - let before_iter = buffer.into_iter().skip_while(move |(_, token)| { - if token.is_separator().is_none() { - total_count -= 1; - } - total_count >= crop_len_before - }); - - // rebalance remaining word count after the match. - let crop_len_after = if crop_marker_before { - crop_len.saturating_sub(crop_len_before + 1) - } else { - crop_len.saturating_sub(total_count + 1) - }; - - let mut taken_after = 0; - let after_iter = tokens.take_while(move |(_, token)| { - let take = taken_after < crop_len_after; - if token.is_separator().is_none() { - taken_after += 1; - } - take - }); - - let iter = before_iter.chain(Some(token)).chain(after_iter); - - Box::new(iter) - } - // If no word matches in the attribute - None => { - let mut count = 0; - let mut tokens = buffer.into_iter(); - let mut out: String = tokens - .by_ref() - .take_while(move |(_, token)| { - let take = count < crop_len; - if token.is_separator().is_none() { - count += 1; - } - take - }) - .map(|(word, _)| word) - .collect(); - - // if there are remaining tokens after formatted interval, - // put a crop marker at the end. - if tokens.next().is_some() { - out.push_str(&self.crop_marker); - } - - return out; - } - } - } - _ => Box::new(tokens.by_ref()), - }; - - let out = if crop_marker_before { - self.crop_marker.clone() - } else { - String::new() - }; - - let mut out = tokens_interval.fold(out, |mut out, (word, token)| { - // Check if we need to do highlighting or computed matches before calling - // Matcher::match since the call is expensive. - if format_options.highlight && token.is_word() { - if let Some(length) = matcher.matches(&token) { - match word.get(..length).zip(word.get(length..)) { - Some((head, tail)) => { - out.push_str(&self.highlight_tags.0); - out.push_str(head); - out.push_str(&self.highlight_tags.1); - out.push_str(tail); - } - // if we are in the middle of a character - // or if all the word should be highlighted, - // we highlight the complete word. - None => { - out.push_str(&self.highlight_tags.0); - out.push_str(word); - out.push_str(&self.highlight_tags.1); - } - } - return out; - } - } - out.push_str(word); - out - }); - - // if there are remaining tokens after formatted interval, - // put a crop marker at the end. - if tokens.next().is_some() { - out.push_str(&self.crop_marker); - } - - out + value => value, } } @@ -810,740 +597,17 @@ fn parse_filter_array(arr: &[Value]) -> Result> { mod test { use super::*; - #[test] - fn no_ids_no_formatted() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - fields.insert("test").unwrap(); - - let document: serde_json::Value = json!({ - "test": "hello", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let formatted_options = BTreeMap::new(); - - let matching_words = MatchingWords::default(); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert!(value.is_empty()); - } - - #[test] - fn formatted_with_highlight_in_word() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let document: serde_json::Value = json!({ - "title": "The Hobbit", - "author": "J. R. R. Tolkien", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: true, - crop: None, - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: None, - }, - ); - - let mut matching_words = BTreeMap::new(); - matching_words.insert("hobbit", Some(3)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "The Hobbit"); - assert_eq!(value["author"], "J. R. R. Tolkien"); - } - - #[test] - fn formatted_with_highlight_in_number() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - let publication_year = fields.insert("publication_year").unwrap(); - - let document: serde_json::Value = json!({ - "title": "The Hobbit", - "author": "J. R. R. Tolkien", - "publication_year": 1937, - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: false, - crop: None, - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: None, - }, - ); - formatted_options.insert( - publication_year, - FormatOptions { - highlight: true, - crop: None, - }, - ); - - let mut matching_words = BTreeMap::new(); - matching_words.insert("1937", Some(4)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "The Hobbit"); - assert_eq!(value["author"], "J. R. R. Tolkien"); - assert_eq!(value["publication_year"], "1937"); - } - - /// https://github.com/meilisearch/meilisearch/issues/1368 - #[test] - fn formatted_with_highlight_emoji() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let document: serde_json::Value = json!({ - "title": "Go💼od luck.", - "author": "JacobLey", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: true, - crop: None, - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: None, - }, - ); - - let mut matching_words = BTreeMap::new(); - // emojis are deunicoded during tokenization - // TODO Tokenizer should remove spaces after deunicode - matching_words.insert("gobriefcase od", Some(11)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "Go💼od luck."); - assert_eq!(value["author"], "JacobLey"); - } - - #[test] - fn formatted_with_highlight_in_unicode_word() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let document: serde_json::Value = json!({ - "title": "étoile", - "author": "J. R. R. Tolkien", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: true, - crop: None, - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: None, - }, - ); - - let mut matching_words = BTreeMap::new(); - matching_words.insert("etoile", Some(1)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "étoile"); - assert_eq!(value["author"], "J. R. R. Tolkien"); - } - - #[test] - fn formatted_with_crop_2() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let document: serde_json::Value = json!({ - "title": "Harry Potter and the Half-Blood Prince", - "author": "J. K. Rowling", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: false, - crop: Some(2), - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: None, - }, - ); - - let mut matching_words = BTreeMap::new(); - matching_words.insert("potter", Some(3)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "Harry Potter…"); - assert_eq!(value["author"], "J. K. Rowling"); - } - - #[test] - fn formatted_with_crop_5() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let document: serde_json::Value = json!({ - "title": "Harry Potter and the Half-Blood Prince", - "author": "J. K. Rowling", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: false, - crop: Some(5), - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: None, - }, - ); - - let mut matching_words = BTreeMap::new(); - matching_words.insert("potter", Some(5)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "Harry Potter and the Half…"); - assert_eq!(value["author"], "J. K. Rowling"); - } - - #[test] - fn formatted_with_crop_0() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let document: serde_json::Value = json!({ - "title": "Harry Potter and the Half-Blood Prince", - "author": "J. K. Rowling", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: false, - crop: Some(0), - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: None, - }, - ); - - let mut matching_words = BTreeMap::new(); - matching_words.insert("potter", Some(6)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "Harry Potter and the Half-Blood Prince"); - assert_eq!(value["author"], "J. K. Rowling"); - } - - #[test] - fn formatted_with_crop_and_no_match() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let document: serde_json::Value = json!({ - "title": "Harry Potter and the Half-Blood Prince", - "author": "J. K. Rowling", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: false, - crop: Some(1), - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: Some(20), - }, - ); - - let mut matching_words = BTreeMap::new(); - matching_words.insert("rowling", Some(3)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "Harry…"); - assert_eq!(value["author"], "J. K. Rowling"); - } - - #[test] - fn formatted_with_crop_and_highlight() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let document: serde_json::Value = json!({ - "title": "Harry Potter and the Half-Blood Prince", - "author": "J. K. Rowling", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: true, - crop: Some(1), - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: None, - }, - ); - - let mut matching_words = BTreeMap::new(); - matching_words.insert("and", Some(3)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "…and…"); - assert_eq!(value["author"], "J. K. Rowling"); - } - - #[test] - fn formatted_with_crop_and_highlight_in_word() { - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - let formatter = Formatter::new( - &analyzer, - (String::from(""), String::from("")), - String::from("…"), - ); - - let mut fields = FieldsIdsMap::new(); - let title = fields.insert("title").unwrap(); - let author = fields.insert("author").unwrap(); - - let document: serde_json::Value = json!({ - "title": "Harry Potter and the Half-Blood Prince", - "author": "J. K. Rowling", - }); - - // we need to convert the `serde_json::Map` into an `IndexMap`. - let document = document - .as_object() - .unwrap() - .into_iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - let mut formatted_options = BTreeMap::new(); - formatted_options.insert( - title, - FormatOptions { - highlight: true, - crop: Some(4), - }, - ); - formatted_options.insert( - author, - FormatOptions { - highlight: false, - crop: None, - }, - ); - - let mut matching_words = BTreeMap::new(); - matching_words.insert("blood", Some(3)); - - let value = format_fields( - &document, - &fields, - &formatter, - &matching_words, - &formatted_options, - ) - .unwrap(); - - assert_eq!(value["title"], "…the Half-Blood Prince"); - assert_eq!(value["author"], "J. K. Rowling"); - } - - #[test] - fn test_compute_value_matches() { - let text = "Call me Ishmael. Some years ago—never mind how long precisely—having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world."; - let value = serde_json::json!(text); - - let mut matcher = BTreeMap::new(); - matcher.insert("ishmael", Some(3)); - matcher.insert("little", Some(6)); - matcher.insert("particular", Some(1)); - - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - - let mut infos = Vec::new(); - - compute_value_matches(&mut infos, &value, &matcher, &analyzer); - - let mut infos = infos.into_iter(); - let crop = |info: MatchInfo| &text[info.start..info.start + info.length]; - - assert_eq!(crop(infos.next().unwrap()), "Ish"); - assert_eq!(crop(infos.next().unwrap()), "little"); - assert_eq!(crop(infos.next().unwrap()), "p"); - assert_eq!(crop(infos.next().unwrap()), "little"); - assert!(infos.next().is_none()); - } - - #[test] - fn test_compute_match() { - let value = serde_json::from_str(r#"{ - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "price": 3.5, - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis Laboris . Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n" - }"#).unwrap(); - let mut matcher = BTreeMap::new(); - matcher.insert("green", Some(5)); - matcher.insert("mollit", Some(6)); - matcher.insert("laboris", Some(7)); - matcher.insert("3", Some(1)); - - let stop_words = fst::Set::default(); - let mut config = AnalyzerConfig::default(); - config.stop_words(&stop_words); - let analyzer = Analyzer::new(config); - - let matches = compute_matches(&matcher, &value, &analyzer); - assert_eq!( - format!("{:?}", matches), - r##"{"about": [MatchInfo { start: 0, length: 6 }, MatchInfo { start: 31, length: 7 }, MatchInfo { start: 191, length: 7 }, MatchInfo { start: 225, length: 7 }, MatchInfo { start: 233, length: 6 }], "color": [MatchInfo { start: 0, length: 5 }], "price": [MatchInfo { start: 0, length: 1 }]}"## - ); - } - #[test] fn test_insert_geo_distance() { let value: Document = serde_json::from_str( r#"{ - "_geo": { - "lat": 50.629973371633746, - "lng": 3.0569447399419567 - }, - "city": "Lille", - "id": "1" - }"#, + "_geo": { + "lat": 50.629973371633746, + "lng": 3.0569447399419567 + }, + "city": "Lille", + "id": "1" + }"#, ) .unwrap(); diff --git a/meilisearch-lib/src/index/updates.rs b/meilisearch-lib/src/index/updates.rs index 3aefa1f5e..07695af05 100644 --- a/meilisearch-lib/src/index/updates.rs +++ b/meilisearch-lib/src/index/updates.rs @@ -68,6 +68,27 @@ pub struct TypoSettings { #[serde(default, skip_serializing_if = "Setting::is_not_set")] pub disable_on_attributes: Setting>, } + +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct FacetingSettings { + #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + pub max_values_per_facet: Setting, +} + +#[cfg_attr(test, derive(proptest_derive::Arbitrary))] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct PaginationSettings { + #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + pub max_total_hits: Setting, +} + /// Holds all the settings for an index. `T` can either be `Checked` if they represents settings /// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a /// call to `check` will return a `Settings` from a `Settings`. @@ -114,6 +135,12 @@ pub struct Settings { #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] pub typo_tolerance: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] + pub faceting: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] + pub pagination: Setting, #[serde(skip)] pub _kind: PhantomData, @@ -131,6 +158,8 @@ impl Settings { synonyms: Setting::Reset, distinct_attribute: Setting::Reset, typo_tolerance: Setting::Reset, + faceting: Setting::Reset, + pagination: Setting::Reset, _kind: PhantomData, } } @@ -146,6 +175,8 @@ impl Settings { synonyms, distinct_attribute, typo_tolerance, + faceting, + pagination, .. } = self; @@ -159,6 +190,8 @@ impl Settings { synonyms, distinct_attribute, typo_tolerance, + faceting, + pagination, _kind: PhantomData, } } @@ -198,6 +231,8 @@ impl Settings { synonyms: self.synonyms, distinct_attribute: self.distinct_attribute, typo_tolerance: self.typo_tolerance, + faceting: self.faceting, + pagination: self.pagination, _kind: PhantomData, } } @@ -427,6 +462,26 @@ pub fn apply_settings_to_builder( } Setting::NotSet => (), } + + match settings.faceting { + Setting::Set(ref value) => match value.max_values_per_facet { + Setting::Set(val) => builder.set_max_values_per_facet(val), + Setting::Reset => builder.reset_max_values_per_facet(), + Setting::NotSet => (), + }, + Setting::Reset => builder.reset_max_values_per_facet(), + Setting::NotSet => (), + } + + match settings.pagination { + Setting::Set(ref value) => match value.max_total_hits { + Setting::Set(val) => builder.set_pagination_max_total_hits(val), + Setting::Reset => builder.reset_pagination_max_total_hits(), + Setting::NotSet => (), + }, + Setting::Reset => builder.reset_pagination_max_total_hits(), + Setting::NotSet => (), + } } #[cfg(test)] @@ -456,6 +511,8 @@ pub(crate) mod test { synonyms: Setting::NotSet, distinct_attribute: Setting::NotSet, typo_tolerance: Setting::NotSet, + faceting: Setting::NotSet, + pagination: Setting::NotSet, _kind: PhantomData::, }; @@ -478,6 +535,8 @@ pub(crate) mod test { synonyms: Setting::NotSet, distinct_attribute: Setting::NotSet, typo_tolerance: Setting::NotSet, + faceting: Setting::NotSet, + pagination: Setting::NotSet, _kind: PhantomData::, }; diff --git a/meilisearch-lib/src/index_controller/dump_actor/actor.rs b/meilisearch-lib/src/index_controller/dump_actor/actor.rs deleted file mode 100644 index 48fc077ca..000000000 --- a/meilisearch-lib/src/index_controller/dump_actor/actor.rs +++ /dev/null @@ -1,191 +0,0 @@ -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use async_stream::stream; -use futures::{lock::Mutex, stream::StreamExt}; -use log::{error, trace}; -use time::macros::format_description; -use time::OffsetDateTime; -use tokio::sync::{mpsc, oneshot, RwLock}; - -use super::error::{DumpActorError, Result}; -use super::{DumpInfo, DumpJob, DumpMsg, DumpStatus}; -use crate::tasks::Scheduler; -use crate::update_file_store::UpdateFileStore; - -pub const CONCURRENT_DUMP_MSG: usize = 10; - -pub struct DumpActor { - inbox: Option>, - update_file_store: UpdateFileStore, - scheduler: Arc>, - dump_path: PathBuf, - analytics_path: PathBuf, - lock: Arc>, - dump_infos: Arc>>, - update_db_size: usize, - index_db_size: usize, -} - -/// Generate uid from creation date -fn generate_uid() -> String { - OffsetDateTime::now_utc() - .format(format_description!( - "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" - )) - .unwrap() -} - -impl DumpActor { - pub fn new( - inbox: mpsc::Receiver, - update_file_store: UpdateFileStore, - scheduler: Arc>, - dump_path: impl AsRef, - analytics_path: impl AsRef, - index_db_size: usize, - update_db_size: usize, - ) -> Self { - let dump_infos = Arc::new(RwLock::new(HashMap::new())); - let lock = Arc::new(Mutex::new(())); - Self { - inbox: Some(inbox), - scheduler, - update_file_store, - dump_path: dump_path.as_ref().into(), - analytics_path: analytics_path.as_ref().into(), - dump_infos, - lock, - index_db_size, - update_db_size, - } - } - - pub async fn run(mut self) { - trace!("Started dump actor."); - - let mut inbox = self - .inbox - .take() - .expect("Dump Actor must have a inbox at this point."); - - let stream = stream! { - loop { - match inbox.recv().await { - Some(msg) => yield msg, - None => break, - } - } - }; - - stream - .for_each_concurrent(Some(CONCURRENT_DUMP_MSG), |msg| self.handle_message(msg)) - .await; - - error!("Dump actor stopped."); - } - - async fn handle_message(&self, msg: DumpMsg) { - use DumpMsg::*; - - match msg { - CreateDump { ret } => { - let _ = self.handle_create_dump(ret).await; - } - DumpInfo { ret, uid } => { - let _ = ret.send(self.handle_dump_info(uid).await); - } - } - } - - async fn handle_create_dump(&self, ret: oneshot::Sender>) { - let uid = generate_uid(); - let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress); - - let _lock = match self.lock.try_lock() { - Some(lock) => lock, - None => { - ret.send(Err(DumpActorError::DumpAlreadyRunning)) - .expect("Dump actor is dead"); - return; - } - }; - - self.dump_infos - .write() - .await - .insert(uid.clone(), info.clone()); - - ret.send(Ok(info)).expect("Dump actor is dead"); - - let task = DumpJob { - dump_path: self.dump_path.clone(), - db_path: self.analytics_path.clone(), - update_file_store: self.update_file_store.clone(), - scheduler: self.scheduler.clone(), - uid: uid.clone(), - update_db_size: self.update_db_size, - index_db_size: self.index_db_size, - }; - - let task_result = tokio::task::spawn_local(task.run()).await; - - let mut dump_infos = self.dump_infos.write().await; - let dump_infos = dump_infos - .get_mut(&uid) - .expect("dump entry deleted while lock was acquired"); - - match task_result { - Ok(Ok(())) => { - dump_infos.done(); - trace!("Dump succeed"); - } - Ok(Err(e)) => { - dump_infos.with_error(e.to_string()); - error!("Dump failed: {}", e); - } - Err(_) => { - dump_infos.with_error("Unexpected error while performing dump.".to_string()); - error!("Dump panicked. Dump status set to failed"); - } - }; - } - - async fn handle_dump_info(&self, uid: String) -> Result { - match self.dump_infos.read().await.get(&uid) { - Some(info) => Ok(info.clone()), - _ => Err(DumpActorError::DumpDoesNotExist(uid)), - } - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_generate_uid() { - let current = OffsetDateTime::now_utc(); - - let uid = generate_uid(); - let (date, time) = uid.split_once('-').unwrap(); - - let date = time::Date::parse( - date, - &format_description!("[year repr:full][month repr:numerical][day padding:zero]"), - ) - .unwrap(); - let time = time::Time::parse( - time, - &format_description!( - "[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" - ), - ) - .unwrap(); - let datetime = time::PrimitiveDateTime::new(date, time); - let datetime = datetime.assume_utc(); - - assert!(current - datetime < time::Duration::SECOND); - } -} diff --git a/meilisearch-lib/src/index_controller/dump_actor/error.rs b/meilisearch-lib/src/index_controller/dump_actor/error.rs deleted file mode 100644 index f72b6d1dd..000000000 --- a/meilisearch-lib/src/index_controller/dump_actor/error.rs +++ /dev/null @@ -1,41 +0,0 @@ -use meilisearch_auth::error::AuthControllerError; -use meilisearch_error::{internal_error, Code, ErrorCode}; - -use crate::{index_resolver::error::IndexResolverError, tasks::error::TaskError}; - -pub type Result = std::result::Result; - -#[derive(thiserror::Error, Debug)] -pub enum DumpActorError { - #[error("A dump is already processing. You must wait until the current process is finished before requesting another dump.")] - DumpAlreadyRunning, - #[error("Dump `{0}` not found.")] - DumpDoesNotExist(String), - #[error("An internal error has occurred. `{0}`.")] - Internal(Box), - #[error("{0}")] - IndexResolver(#[from] IndexResolverError), -} - -internal_error!( - DumpActorError: milli::heed::Error, - std::io::Error, - tokio::task::JoinError, - tokio::sync::oneshot::error::RecvError, - serde_json::error::Error, - tempfile::PersistError, - fs_extra::error::Error, - AuthControllerError, - TaskError -); - -impl ErrorCode for DumpActorError { - fn error_code(&self) -> Code { - match self { - DumpActorError::DumpAlreadyRunning => Code::DumpAlreadyInProgress, - DumpActorError::DumpDoesNotExist(_) => Code::DumpNotFound, - DumpActorError::Internal(_) => Code::Internal, - DumpActorError::IndexResolver(e) => e.error_code(), - } - } -} diff --git a/meilisearch-lib/src/index_controller/dump_actor/handle_impl.rs b/meilisearch-lib/src/index_controller/dump_actor/handle_impl.rs deleted file mode 100644 index 16a312e70..000000000 --- a/meilisearch-lib/src/index_controller/dump_actor/handle_impl.rs +++ /dev/null @@ -1,26 +0,0 @@ -use tokio::sync::{mpsc, oneshot}; - -use super::error::Result; -use super::{DumpActorHandle, DumpInfo, DumpMsg}; - -#[derive(Clone)] -pub struct DumpActorHandleImpl { - pub sender: mpsc::Sender, -} - -#[async_trait::async_trait] -impl DumpActorHandle for DumpActorHandleImpl { - async fn create_dump(&self) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = DumpMsg::CreateDump { ret }; - let _ = self.sender.send(msg).await; - receiver.await.expect("IndexActor has been killed") - } - - async fn dump_info(&self, uid: String) -> Result { - let (ret, receiver) = oneshot::channel(); - let msg = DumpMsg::DumpInfo { ret, uid }; - let _ = self.sender.send(msg).await; - receiver.await.expect("IndexActor has been killed") - } -} diff --git a/meilisearch-lib/src/index_controller/dump_actor/message.rs b/meilisearch-lib/src/index_controller/dump_actor/message.rs deleted file mode 100644 index 6c9dded9f..000000000 --- a/meilisearch-lib/src/index_controller/dump_actor/message.rs +++ /dev/null @@ -1,14 +0,0 @@ -use tokio::sync::oneshot; - -use super::error::Result; -use super::DumpInfo; - -pub enum DumpMsg { - CreateDump { - ret: oneshot::Sender>, - }, - DumpInfo { - uid: String, - ret: oneshot::Sender>, - }, -} diff --git a/meilisearch-lib/src/index_controller/dump_actor/mod.rs b/meilisearch-lib/src/index_controller/dump_actor/mod.rs deleted file mode 100644 index 16e328e3b..000000000 --- a/meilisearch-lib/src/index_controller/dump_actor/mod.rs +++ /dev/null @@ -1,515 +0,0 @@ -use std::fs::File; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use anyhow::bail; -use log::{info, trace}; -use serde::{Deserialize, Serialize}; -use time::OffsetDateTime; - -pub use actor::DumpActor; -pub use handle_impl::*; -use meilisearch_auth::AuthController; -pub use message::DumpMsg; -use tempfile::TempDir; -use tokio::fs::create_dir_all; -use tokio::sync::{oneshot, RwLock}; - -use crate::analytics; -use crate::compression::{from_tar_gz, to_tar_gz}; -use crate::index_controller::dump_actor::error::DumpActorError; -use crate::index_controller::dump_actor::loaders::{v2, v3, v4}; -use crate::options::IndexerOpts; -use crate::tasks::task::Job; -use crate::tasks::Scheduler; -use crate::update_file_store::UpdateFileStore; -use error::Result; - -mod actor; -mod compat; -pub mod error; -mod handle_impl; -mod loaders; -mod message; - -const META_FILE_NAME: &str = "metadata.json"; - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct Metadata { - db_version: String, - index_db_size: usize, - update_db_size: usize, - #[serde(with = "time::serde::rfc3339")] - dump_date: OffsetDateTime, -} - -impl Metadata { - pub fn new(index_db_size: usize, update_db_size: usize) -> Self { - Self { - db_version: env!("CARGO_PKG_VERSION").to_string(), - index_db_size, - update_db_size, - dump_date: OffsetDateTime::now_utc(), - } - } -} - -#[async_trait::async_trait] -#[cfg_attr(test, mockall::automock)] -pub trait DumpActorHandle { - /// Start the creation of a dump - /// Implementation: [handle_impl::DumpActorHandleImpl::create_dump] - async fn create_dump(&self) -> Result; - - /// Return the status of an already created dump - /// Implementation: [handle_impl::DumpActorHandleImpl::dump_info] - async fn dump_info(&self, uid: String) -> Result; -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct MetadataV1 { - pub db_version: String, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(tag = "dumpVersion")] -pub enum MetadataVersion { - V1(MetadataV1), - V2(Metadata), - V3(Metadata), - V4(Metadata), -} - -impl MetadataVersion { - pub fn load_dump( - self, - src: impl AsRef, - dst: impl AsRef, - index_db_size: usize, - meta_env_size: usize, - indexing_options: &IndexerOpts, - ) -> anyhow::Result<()> { - match self { - MetadataVersion::V1(_meta) => { - anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.") - } - MetadataVersion::V2(meta) => v2::load_dump( - meta, - src, - dst, - index_db_size, - meta_env_size, - indexing_options, - )?, - MetadataVersion::V3(meta) => v3::load_dump( - meta, - src, - dst, - index_db_size, - meta_env_size, - indexing_options, - )?, - MetadataVersion::V4(meta) => v4::load_dump( - meta, - src, - dst, - index_db_size, - meta_env_size, - indexing_options, - )?, - } - - Ok(()) - } - - pub fn new_v4(index_db_size: usize, update_db_size: usize) -> Self { - let meta = Metadata::new(index_db_size, update_db_size); - Self::V4(meta) - } - - pub fn db_version(&self) -> &str { - match self { - Self::V1(meta) => &meta.db_version, - Self::V2(meta) | Self::V3(meta) | Self::V4(meta) => &meta.db_version, - } - } - - pub fn version(&self) -> &str { - match self { - MetadataVersion::V1(_) => "V1", - MetadataVersion::V2(_) => "V2", - MetadataVersion::V3(_) => "V3", - MetadataVersion::V4(_) => "V4", - } - } - - pub fn dump_date(&self) -> Option<&OffsetDateTime> { - match self { - MetadataVersion::V1(_) => None, - MetadataVersion::V2(meta) | MetadataVersion::V3(meta) | MetadataVersion::V4(meta) => { - Some(&meta.dump_date) - } - } - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] -#[serde(rename_all = "snake_case")] -pub enum DumpStatus { - Done, - InProgress, - Failed, -} - -#[derive(Debug, Serialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct DumpInfo { - pub uid: String, - pub status: DumpStatus, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - #[serde(with = "time::serde::rfc3339")] - started_at: OffsetDateTime, - #[serde( - skip_serializing_if = "Option::is_none", - with = "time::serde::rfc3339::option" - )] - finished_at: Option, -} - -impl DumpInfo { - pub fn new(uid: String, status: DumpStatus) -> Self { - Self { - uid, - status, - error: None, - started_at: OffsetDateTime::now_utc(), - finished_at: None, - } - } - - pub fn with_error(&mut self, error: String) { - self.status = DumpStatus::Failed; - self.finished_at = Some(OffsetDateTime::now_utc()); - self.error = Some(error); - } - - pub fn done(&mut self) { - self.finished_at = Some(OffsetDateTime::now_utc()); - self.status = DumpStatus::Done; - } - - pub fn dump_already_in_progress(&self) -> bool { - self.status == DumpStatus::InProgress - } -} - -pub fn load_dump( - dst_path: impl AsRef, - src_path: impl AsRef, - ignore_dump_if_db_exists: bool, - ignore_missing_dump: bool, - index_db_size: usize, - update_db_size: usize, - indexer_opts: &IndexerOpts, -) -> anyhow::Result<()> { - let empty_db = crate::is_empty_db(&dst_path); - let src_path_exists = src_path.as_ref().exists(); - - if empty_db && src_path_exists { - let (tmp_src, tmp_dst, meta) = extract_dump(&dst_path, &src_path)?; - meta.load_dump( - tmp_src.path(), - tmp_dst.path(), - index_db_size, - update_db_size, - indexer_opts, - )?; - persist_dump(&dst_path, tmp_dst)?; - Ok(()) - } else if !empty_db && !ignore_dump_if_db_exists { - bail!( - "database already exists at {:?}, try to delete it or rename it", - dst_path - .as_ref() - .canonicalize() - .unwrap_or_else(|_| dst_path.as_ref().to_owned()) - ) - } else if !src_path_exists && !ignore_missing_dump { - bail!("dump doesn't exist at {:?}", src_path.as_ref()) - } else { - // there is nothing to do - Ok(()) - } -} - -fn extract_dump( - dst_path: impl AsRef, - src_path: impl AsRef, -) -> anyhow::Result<(TempDir, TempDir, MetadataVersion)> { - // Setup a temp directory path in the same path as the database, to prevent cross devices - // references. - let temp_path = dst_path - .as_ref() - .parent() - .map(ToOwned::to_owned) - .unwrap_or_else(|| ".".into()); - if cfg!(windows) { - std::env::set_var("TMP", temp_path); - } else { - std::env::set_var("TMPDIR", temp_path); - } - - let tmp_src = tempfile::tempdir()?; - let tmp_src_path = tmp_src.path(); - - from_tar_gz(&src_path, tmp_src_path)?; - - let meta_path = tmp_src_path.join(META_FILE_NAME); - let mut meta_file = File::open(&meta_path)?; - let meta: MetadataVersion = serde_json::from_reader(&mut meta_file)?; - - if !dst_path.as_ref().exists() { - std::fs::create_dir_all(dst_path.as_ref())?; - } - - let tmp_dst = tempfile::tempdir_in(dst_path.as_ref())?; - - info!( - "Loading dump {}, dump database version: {}, dump version: {}", - meta.dump_date() - .map(|t| format!("from {}", t)) - .unwrap_or_else(String::new), - meta.db_version(), - meta.version() - ); - - Ok((tmp_src, tmp_dst, meta)) -} - -fn persist_dump(dst_path: impl AsRef, tmp_dst: TempDir) -> anyhow::Result<()> { - let persisted_dump = tmp_dst.into_path(); - - // Delete everything in the `data.ms` except the tempdir. - if dst_path.as_ref().exists() { - for file in dst_path.as_ref().read_dir().unwrap() { - let file = file.unwrap().path(); - if file.file_name() == persisted_dump.file_name() { - continue; - } - - if file.is_file() { - std::fs::remove_file(&file)?; - } else { - std::fs::remove_dir_all(&file)?; - } - } - } - - // Move the whole content of the tempdir into the `data.ms`. - for file in persisted_dump.read_dir().unwrap() { - let file = file.unwrap().path(); - - std::fs::rename(&file, &dst_path.as_ref().join(file.file_name().unwrap()))?; - } - - // Delete the empty tempdir. - std::fs::remove_dir_all(&persisted_dump)?; - - Ok(()) -} - -struct DumpJob { - dump_path: PathBuf, - db_path: PathBuf, - update_file_store: UpdateFileStore, - scheduler: Arc>, - uid: String, - update_db_size: usize, - index_db_size: usize, -} - -impl DumpJob { - async fn run(self) -> Result<()> { - trace!("Performing dump."); - - create_dir_all(&self.dump_path).await?; - - let temp_dump_dir = tokio::task::spawn_blocking(tempfile::TempDir::new).await??; - let temp_dump_path = temp_dump_dir.path().to_owned(); - - let meta = MetadataVersion::new_v4(self.index_db_size, self.update_db_size); - let meta_path = temp_dump_path.join(META_FILE_NAME); - let mut meta_file = File::create(&meta_path)?; - serde_json::to_writer(&mut meta_file, &meta)?; - analytics::copy_user_id(&self.db_path, &temp_dump_path); - - create_dir_all(&temp_dump_path.join("indexes")).await?; - - let (sender, receiver) = oneshot::channel(); - - self.scheduler - .write() - .await - .schedule_job(Job::Dump { - ret: sender, - path: temp_dump_path.clone(), - }) - .await; - - // wait until the job has started performing before finishing the dump process - let sender = receiver.await??; - - AuthController::dump(&self.db_path, &temp_dump_path)?; - - //TODO(marin): this is not right, the scheduler should dump itself, not do it here... - self.scheduler - .read() - .await - .dump(&temp_dump_path, self.update_file_store.clone()) - .await?; - - let dump_path = tokio::task::spawn_blocking(move || -> Result { - // for now we simply copy the updates/updates_files - // FIXME: We may copy more files than necessary, if new files are added while we are - // performing the dump. We need a way to filter them out. - - let temp_dump_file = tempfile::NamedTempFile::new_in(&self.dump_path)?; - to_tar_gz(temp_dump_path, temp_dump_file.path()) - .map_err(|e| DumpActorError::Internal(e.into()))?; - - let dump_path = self.dump_path.join(self.uid).with_extension("dump"); - temp_dump_file.persist(&dump_path)?; - - Ok(dump_path) - }) - .await??; - - // notify the update loop that we are finished performing the dump. - let _ = sender.send(()); - - info!("Created dump in {:?}.", dump_path); - - Ok(()) - } -} - -#[cfg(test)] -mod test { - use nelson::Mocker; - use once_cell::sync::Lazy; - - use super::*; - use crate::index_resolver::error::IndexResolverError; - use crate::options::SchedulerConfig; - use crate::tasks::error::Result as TaskResult; - use crate::tasks::task::{Task, TaskId}; - use crate::tasks::{MockTaskPerformer, TaskFilter, TaskStore}; - use crate::update_file_store::UpdateFileStore; - - fn setup() { - static SETUP: Lazy<()> = Lazy::new(|| { - if cfg!(windows) { - std::env::set_var("TMP", "."); - } else { - std::env::set_var("TMPDIR", "."); - } - }); - - // just deref to make sure the env is setup - *SETUP - } - - #[actix_rt::test] - async fn test_dump_normal() { - setup(); - - let tmp = tempfile::tempdir().unwrap(); - - let mocker = Mocker::default(); - let update_file_store = UpdateFileStore::mock(mocker); - - let mut performer = MockTaskPerformer::new(); - performer - .expect_process_job() - .once() - .returning(|j| match j { - Job::Dump { ret, .. } => { - let (sender, _receiver) = oneshot::channel(); - ret.send(Ok(sender)).unwrap(); - } - _ => unreachable!(), - }); - let performer = Arc::new(performer); - let mocker = Mocker::default(); - mocker - .when::<(&Path, UpdateFileStore), TaskResult<()>>("dump") - .then(|_| Ok(())); - mocker - .when::<(Option, Option, Option), TaskResult>>( - "list_tasks", - ) - .then(|_| Ok(Vec::new())); - let store = TaskStore::mock(mocker); - let config = SchedulerConfig::default(); - - let scheduler = Scheduler::new(store, performer, config).unwrap(); - - let task = DumpJob { - dump_path: tmp.path().into(), - // this should do nothing - update_file_store, - db_path: tmp.path().into(), - uid: String::from("test"), - update_db_size: 4096 * 10, - index_db_size: 4096 * 10, - scheduler, - }; - - task.run().await.unwrap(); - } - - #[actix_rt::test] - async fn error_performing_dump() { - let tmp = tempfile::tempdir().unwrap(); - - let mocker = Mocker::default(); - let file_store = UpdateFileStore::mock(mocker); - - let mocker = Mocker::default(); - mocker - .when::<(Option, Option, Option), TaskResult>>( - "list_tasks", - ) - .then(|_| Ok(Vec::new())); - let task_store = TaskStore::mock(mocker); - let mut performer = MockTaskPerformer::new(); - performer - .expect_process_job() - .once() - .returning(|job| match job { - Job::Dump { ret, .. } => drop(ret.send(Err(IndexResolverError::BadlyFormatted( - "blabla".to_string(), - )))), - _ => unreachable!(), - }); - let performer = Arc::new(performer); - - let scheduler = Scheduler::new(task_store, performer, SchedulerConfig::default()).unwrap(); - - let task = DumpJob { - dump_path: tmp.path().into(), - // this should do nothing - db_path: tmp.path().into(), - update_file_store: file_store, - uid: String::from("test"), - update_db_size: 4096 * 10, - index_db_size: 4096 * 10, - scheduler, - }; - - assert!(task.run().await.is_err()); - } -} diff --git a/meilisearch-lib/src/index_controller/error.rs b/meilisearch-lib/src/index_controller/error.rs index 85af76623..ab2dd142d 100644 --- a/meilisearch-lib/src/index_controller/error.rs +++ b/meilisearch-lib/src/index_controller/error.rs @@ -1,16 +1,17 @@ use std::error::Error; -use meilisearch_error::Code; -use meilisearch_error::{internal_error, ErrorCode}; +use meilisearch_types::error::{Code, ErrorCode}; +use meilisearch_types::index_uid::IndexUidFormatError; +use meilisearch_types::internal_error; use tokio::task::JoinError; use super::DocumentAdditionFormat; use crate::document_formats::DocumentFormatError; +use crate::dump::error::DumpError; use crate::index::error::IndexError; use crate::tasks::error::TaskError; use crate::update_file_store::UpdateFileStoreError; -use super::dump_actor::error::DumpActorError; use crate::index_resolver::error::IndexResolverError; pub type Result = std::result::Result; @@ -28,7 +29,7 @@ pub enum IndexControllerError { #[error("{0}")] TaskError(#[from] TaskError), #[error("{0}")] - DumpError(#[from] DumpActorError), + DumpError(#[from] DumpError), #[error("{0}")] DocumentFormatError(#[from] DocumentFormatError), #[error("A {0} payload is missing.")] @@ -63,3 +64,9 @@ impl ErrorCode for IndexControllerError { } } } + +impl From for IndexControllerError { + fn from(err: IndexUidFormatError) -> Self { + IndexResolverError::from(err).into() + } +} diff --git a/meilisearch-lib/src/index_controller/mod.rs b/meilisearch-lib/src/index_controller/mod.rs index 77ff2621b..88782c5ea 100644 --- a/meilisearch-lib/src/index_controller/mod.rs +++ b/meilisearch-lib/src/index_controller/mod.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use std::fmt; use std::io::Cursor; use std::path::{Path, PathBuf}; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; @@ -10,34 +11,36 @@ use actix_web::error::PayloadError; use bytes::Bytes; use futures::Stream; use futures::StreamExt; +use meilisearch_types::index_uid::IndexUid; use milli::update::IndexDocumentsMethod; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; -use tokio::sync::{mpsc, RwLock}; +use tokio::sync::RwLock; use tokio::task::spawn_blocking; use tokio::time::sleep; use uuid::Uuid; use crate::document_formats::{read_csv, read_json, read_ndjson}; +use crate::dump::{self, load_dump, DumpHandler}; use crate::index::{ Checked, Document, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings, Unchecked, }; -use crate::index_controller::dump_actor::{load_dump, DumpActor, DumpActorHandleImpl}; +use crate::index_resolver::error::IndexResolverError; use crate::options::{IndexerOpts, SchedulerConfig}; use crate::snapshot::{load_snapshot, SnapshotService}; use crate::tasks::error::TaskError; use crate::tasks::task::{DocumentDeletion, Task, TaskContent, TaskId}; -use crate::tasks::{Scheduler, TaskFilter, TaskStore}; +use crate::tasks::{ + BatchHandler, EmptyBatchHandler, Scheduler, SnapshotHandler, TaskFilter, TaskStore, +}; use error::Result; -use self::dump_actor::{DumpActorHandle, DumpInfo}; use self::error::IndexControllerError; use crate::index_resolver::index_store::{IndexStore, MapIndexStore}; use crate::index_resolver::meta_store::{HeedMetaStore, IndexMetaStore}; -use crate::index_resolver::{create_index_resolver, IndexResolver, IndexUid}; +use crate::index_resolver::{create_index_resolver, IndexResolver}; use crate::update_file_store::UpdateFileStore; -mod dump_actor; pub mod error; pub mod versioning; @@ -61,7 +64,6 @@ pub struct IndexMetadata { #[serde(skip)] pub uuid: Uuid, pub uid: String, - name: String, #[serde(flatten)] pub meta: IndexMeta, } @@ -73,11 +75,10 @@ pub struct IndexSettings { } pub struct IndexController { - index_resolver: Arc>, + pub index_resolver: Arc>, scheduler: Arc>, task_store: TaskStore, - dump_handle: dump_actor::DumpActorHandleImpl, - update_file_store: UpdateFileStore, + pub update_file_store: UpdateFileStore, } /// Need a custom implementation for clone because deriving require that U and I are clone. @@ -86,7 +87,6 @@ impl Clone for IndexController { Self { index_resolver: self.index_resolver.clone(), scheduler: self.scheduler.clone(), - dump_handle: self.dump_handle.clone(), update_file_store: self.update_file_store.clone(), task_store: self.task_store.clone(), } @@ -220,30 +220,30 @@ impl IndexControllerBuilder { update_file_store.clone(), )?); - let task_store = TaskStore::new(meta_env)?; - let scheduler = - Scheduler::new(task_store.clone(), index_resolver.clone(), scheduler_config)?; - let dump_path = self .dump_dst .ok_or_else(|| anyhow::anyhow!("Missing dump directory path"))?; - let dump_handle = { - let analytics_path = &db_path; - let (sender, receiver) = mpsc::channel(10); - let actor = DumpActor::new( - receiver, - update_file_store.clone(), - scheduler.clone(), - dump_path, - analytics_path, - index_size, - task_store_size, - ); - tokio::task::spawn_local(actor.run()); + let dump_handler = Arc::new(DumpHandler::new( + dump_path, + db_path.as_ref().into(), + update_file_store.clone(), + task_store_size, + index_size, + meta_env.clone(), + index_resolver.clone(), + )); + let task_store = TaskStore::new(meta_env)?; - DumpActorHandleImpl { sender } - }; + // register all the batch handlers for use with the scheduler. + let handlers: Vec> = vec![ + index_resolver.clone(), + dump_handler, + Arc::new(SnapshotHandler), + // dummy handler to catch all empty batches + Arc::new(EmptyBatchHandler), + ]; + let scheduler = Scheduler::new(task_store.clone(), handlers, scheduler_config)?; if self.schedule_snapshot { let snapshot_period = self @@ -268,7 +268,6 @@ impl IndexControllerBuilder { Ok(IndexController { index_resolver, scheduler, - dump_handle, update_file_store, task_store, }) @@ -359,12 +358,16 @@ where } pub async fn register_update(&self, uid: String, update: Update) -> Result { - let uid = IndexUid::new(uid)?; + let index_uid = IndexUid::from_str(&uid).map_err(IndexResolverError::from)?; let content = match update { - Update::DeleteDocuments(ids) => { - TaskContent::DocumentDeletion(DocumentDeletion::Ids(ids)) - } - Update::ClearDocuments => TaskContent::DocumentDeletion(DocumentDeletion::Clear), + Update::DeleteDocuments(ids) => TaskContent::DocumentDeletion { + index_uid, + deletion: DocumentDeletion::Ids(ids), + }, + Update::ClearDocuments => TaskContent::DocumentDeletion { + index_uid, + deletion: DocumentDeletion::Clear, + }, Update::Settings { settings, is_deletion, @@ -373,6 +376,7 @@ where settings, is_deletion, allow_index_creation, + index_uid, }, Update::DocumentAddition { mut payload, @@ -412,19 +416,34 @@ where primary_key, documents_count, allow_index_creation, + index_uid, } } - Update::DeleteIndex => TaskContent::IndexDeletion, - Update::CreateIndex { primary_key } => TaskContent::IndexCreation { primary_key }, - Update::UpdateIndex { primary_key } => TaskContent::IndexUpdate { primary_key }, + Update::DeleteIndex => TaskContent::IndexDeletion { index_uid }, + Update::CreateIndex { primary_key } => TaskContent::IndexCreation { + primary_key, + index_uid, + }, + Update::UpdateIndex { primary_key } => TaskContent::IndexUpdate { + primary_key, + index_uid, + }, }; - let task = self.task_store.register(uid, content).await?; + let task = self.task_store.register(content).await?; self.scheduler.read().await.notify(); Ok(task) } + pub async fn register_dump_task(&self) -> Result { + let uid = dump::generate_uid(); + let content = TaskContent::Dump { uid }; + let task = self.task_store.register(content).await?; + self.scheduler.read().await.notify(); + Ok(task) + } + pub async fn get_task(&self, id: TaskId, filter: Option) -> Result { let task = self.scheduler.read().await.get_task(id, filter).await?; Ok(task) @@ -502,7 +521,6 @@ where let meta = index.meta()?; let meta = IndexMetadata { uuid: index.uuid(), - name: uid.clone(), uid, meta, }; @@ -518,18 +536,19 @@ where Ok(settings) } + /// Return the total number of documents contained in the index + the selected documents. pub async fn documents( &self, uid: String, offset: usize, limit: usize, attributes_to_retrieve: Option>, - ) -> Result> { + ) -> Result<(u64, Vec)> { let index = self.index_resolver.get_index(uid).await?; - let documents = + let result = spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve)) .await??; - Ok(documents) + Ok(result) } pub async fn document( @@ -555,12 +574,7 @@ where let index = self.index_resolver.get_index(uid.clone()).await?; let uuid = index.uuid(); let meta = spawn_blocking(move || index.meta()).await??; - let meta = IndexMetadata { - uuid, - name: uid.clone(), - uid, - meta, - }; + let meta = IndexMetadata { uuid, uid, meta }; Ok(meta) } @@ -569,8 +583,7 @@ where // Check if the currently indexing update is from our index. let is_indexing = processing_tasks .first() - .map(|task| task.index_uid.as_str() == uid) - .unwrap_or_default(); + .map_or(false, |task| task.index_uid().map_or(false, |u| u == uid)); let index = self.index_resolver.get_index(uid).await?; let mut stats = spawn_blocking(move || index.stats()).await??; @@ -605,7 +618,7 @@ where // Check if the currently indexing update is from our index. stats.is_indexing = processing_tasks .first() - .map(|p| p.index_uid.as_str() == index_uid) + .and_then(|p| p.index_uid().map(|u| u == index_uid)) .or(Some(false)); indexes.insert(index_uid, stats); @@ -617,14 +630,6 @@ where indexes, }) } - - pub async fn create_dump(&self) -> Result { - Ok(self.dump_handle.create_dump().await?) - } - - pub async fn dump_info(&self, uid: String) -> Result { - Ok(self.dump_handle.dump_info(uid).await?) - } } pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { @@ -649,7 +654,7 @@ mod test { use crate::index::error::Result as IndexResult; use crate::index::Index; use crate::index::{ - default_crop_marker, default_highlight_post_tag, default_highlight_pre_tag, + DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, }; use crate::index_resolver::index_store::MockIndexStore; use crate::index_resolver::meta_store::MockIndexMetaStore; @@ -662,13 +667,11 @@ mod test { index_resolver: Arc>, task_store: TaskStore, update_file_store: UpdateFileStore, - dump_handle: DumpActorHandleImpl, scheduler: Arc>, ) -> Self { IndexController { index_resolver, task_store, - dump_handle, update_file_store, scheduler, } @@ -687,25 +690,23 @@ mod test { attributes_to_crop: None, crop_length: 18, attributes_to_highlight: None, - matches: true, + show_matches_position: true, filter: None, sort: None, - facets_distribution: None, - highlight_pre_tag: default_highlight_pre_tag(), - highlight_post_tag: default_highlight_post_tag(), - crop_marker: default_crop_marker(), + facets: None, + highlight_pre_tag: DEFAULT_HIGHLIGHT_PRE_TAG(), + highlight_post_tag: DEFAULT_HIGHLIGHT_POST_TAG(), + crop_marker: DEFAULT_CROP_MARKER(), }; let result = SearchResult { hits: vec![], - nb_hits: 29, - exhaustive_nb_hits: true, + estimated_total_hits: 29, query: "hello world".to_string(), limit: 24, offset: 0, processing_time_ms: 50, - facets_distribution: None, - exhaustive_facets_count: Some(true), + facet_distribution: None, }; let mut uuid_store = MockIndexMetaStore::new(); @@ -754,19 +755,12 @@ mod test { let task_store = TaskStore::mock(task_store_mocker); let scheduler = Scheduler::new( task_store.clone(), - index_resolver.clone(), + vec![index_resolver.clone()], SchedulerConfig::default(), ) .unwrap(); - let (sender, _) = mpsc::channel(1); - let dump_handle = DumpActorHandleImpl { sender }; - let index_controller = IndexController::mock( - index_resolver, - task_store, - update_file_store, - dump_handle, - scheduler, - ); + let index_controller = + IndexController::mock(index_resolver, task_store, update_file_store, scheduler); let r = index_controller .search(index_uid.to_owned(), query.clone()) diff --git a/meilisearch-lib/src/index_controller/updates/error.rs b/meilisearch-lib/src/index_controller/updates/error.rs index 434783041..7ecaa45c5 100644 --- a/meilisearch-lib/src/index_controller/updates/error.rs +++ b/meilisearch-lib/src/index_controller/updates/error.rs @@ -1,7 +1,7 @@ use std::error::Error; use std::fmt; -use meilisearch_error::{internal_error, Code, ErrorCode}; +use meilisearch_types::{internal_error, Code, ErrorCode}; use crate::{ document_formats::DocumentFormatError, diff --git a/meilisearch-lib/src/index_resolver/error.rs b/meilisearch-lib/src/index_resolver/error.rs index 6c86aa6b8..d973d2229 100644 --- a/meilisearch-lib/src/index_resolver/error.rs +++ b/meilisearch-lib/src/index_resolver/error.rs @@ -1,11 +1,13 @@ use std::fmt; -use meilisearch_error::{internal_error, Code, ErrorCode}; +use meilisearch_types::error::{Code, ErrorCode}; +use meilisearch_types::index_uid::IndexUidFormatError; +use meilisearch_types::internal_error; use tokio::sync::mpsc::error::SendError as MpscSendError; use tokio::sync::oneshot::error::RecvError as OneshotRecvError; use uuid::Uuid; -use crate::{error::MilliError, index::error::IndexError}; +use crate::{error::MilliError, index::error::IndexError, update_file_store::UpdateFileStoreError}; pub type Result = std::result::Result; @@ -25,8 +27,8 @@ pub enum IndexResolverError { UuidAlreadyExists(Uuid), #[error("{0}")] Milli(#[from] milli::Error), - #[error("`{0}` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).")] - BadlyFormatted(String), + #[error("{0}")] + BadlyFormatted(#[from] IndexUidFormatError), } impl From> for IndexResolverError @@ -49,7 +51,8 @@ internal_error!( uuid::Error, std::io::Error, tokio::task::JoinError, - serde_json::Error + serde_json::Error, + UpdateFileStoreError ); impl ErrorCode for IndexResolverError { diff --git a/meilisearch-lib/src/index_resolver/message.rs b/meilisearch-lib/src/index_resolver/message.rs deleted file mode 100644 index 25a0d64a9..000000000 --- a/meilisearch-lib/src/index_resolver/message.rs +++ /dev/null @@ -1,37 +0,0 @@ -use std::{collections::HashSet, path::PathBuf}; - -use tokio::sync::oneshot; -use uuid::Uuid; - -use crate::index::Index; -use super::error::Result; - -pub enum IndexResolverMsg { - Get { - uid: String, - ret: oneshot::Sender>, - }, - Delete { - uid: String, - ret: oneshot::Sender>, - }, - List { - ret: oneshot::Sender>>, - }, - Insert { - uuid: Uuid, - name: String, - ret: oneshot::Sender>, - }, - SnapshotRequest { - path: PathBuf, - ret: oneshot::Sender>>, - }, - GetSize { - ret: oneshot::Sender>, - }, - DumpRequest { - path: PathBuf, - ret: oneshot::Sender>>, - }, -} diff --git a/meilisearch-lib/src/index_resolver/meta_store.rs b/meilisearch-lib/src/index_resolver/meta_store.rs index f53f9cae9..f335d9923 100644 --- a/meilisearch-lib/src/index_resolver/meta_store.rs +++ b/meilisearch-lib/src/index_resolver/meta_store.rs @@ -3,6 +3,7 @@ use std::fs::{create_dir_all, File}; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use walkdir::WalkDir; use milli::heed::types::{SerdeBincode, Str}; use milli::heed::{CompactionOption, Database, Env}; @@ -11,7 +12,6 @@ use uuid::Uuid; use super::error::{IndexResolverError, Result}; use crate::tasks::task::TaskId; -use crate::EnvSizer; #[derive(Serialize, Deserialize)] pub struct DumpEntry { @@ -131,7 +131,12 @@ impl HeedMetaStore { } fn get_size(&self) -> Result { - Ok(self.env.size()) + Ok(WalkDir::new(self.env.path()) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len())) } pub fn dump(&self, path: PathBuf) -> Result<()> { diff --git a/meilisearch-lib/src/index_resolver/mod.rs b/meilisearch-lib/src/index_resolver/mod.rs index 8ca3efdc6..686a549b9 100644 --- a/meilisearch-lib/src/index_resolver/mod.rs +++ b/meilisearch-lib/src/index_resolver/mod.rs @@ -2,38 +2,35 @@ pub mod error; pub mod index_store; pub mod meta_store; -use std::convert::{TryFrom, TryInto}; +use std::convert::TryFrom; use std::path::Path; use std::sync::Arc; use error::{IndexResolverError, Result}; use index_store::{IndexStore, MapIndexStore}; -use meilisearch_error::ResponseError; +use meilisearch_types::error::ResponseError; +use meilisearch_types::index_uid::IndexUid; use meta_store::{HeedMetaStore, IndexMetaStore}; use milli::heed::Env; use milli::update::{DocumentDeletionResult, IndexerConfig}; -use serde::{Deserialize, Serialize}; use time::OffsetDateTime; -use tokio::sync::oneshot; use tokio::task::spawn_blocking; use uuid::Uuid; use crate::index::{error::Result as IndexResult, Index}; use crate::options::IndexerOpts; -use crate::tasks::batch::Batch; -use crate::tasks::task::{DocumentDeletion, Job, Task, TaskContent, TaskEvent, TaskId, TaskResult}; -use crate::tasks::TaskPerformer; +use crate::tasks::task::{DocumentDeletion, Task, TaskContent, TaskEvent, TaskId, TaskResult}; use crate::update_file_store::UpdateFileStore; use self::meta_store::IndexMeta; pub type HardStateIndexResolver = IndexResolver; -/// An index uid is composed of only ascii alphanumeric characters, - and _, between 1 and 400 -/// bytes long -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -#[cfg_attr(test, derive(proptest_derive::Arbitrary))] -pub struct IndexUid(#[cfg_attr(test, proptest(regex("[a-zA-Z0-9_-]{1,400}")))] String); +#[cfg(not(test))] +pub use real::IndexResolver; + +#[cfg(test)] +pub use test::MockIndexResolver as IndexResolver; pub fn create_index_resolver( path: impl AsRef, @@ -47,620 +44,633 @@ pub fn create_index_resolver( Ok(IndexResolver::new(uuid_store, index_store, file_store)) } -impl IndexUid { - pub fn new(uid: String) -> Result { - if !uid - .chars() - .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') - || !(1..=400).contains(&uid.len()) - { - Err(IndexResolverError::BadlyFormatted(uid)) - } else { - Ok(Self(uid)) - } +mod real { + use super::*; + + pub struct IndexResolver { + pub(super) index_uuid_store: U, + pub(super) index_store: I, + pub(super) file_store: UpdateFileStore, } - #[cfg(test)] - pub fn new_unchecked(s: impl AsRef) -> Self { - Self(s.as_ref().to_string()) - } - - pub fn into_inner(self) -> String { - self.0 - } - - /// Return a reference over the inner str. - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl std::ops::Deref for IndexUid { - type Target = str; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl TryInto for String { - type Error = IndexResolverError; - - fn try_into(self) -> Result { - IndexUid::new(self) - } -} - -#[async_trait::async_trait] -impl TaskPerformer for IndexResolver -where - U: IndexMetaStore + Send + Sync + 'static, - I: IndexStore + Send + Sync + 'static, -{ - async fn process_batch(&self, mut batch: Batch) -> Batch { - // If a batch contains multiple tasks, then it must be a document addition batch - if let Some(Task { - content: TaskContent::DocumentAddition { .. }, - .. - }) = batch.tasks.first() - { - debug_assert!(batch.tasks.iter().all(|t| matches!( - t, - Task { - content: TaskContent::DocumentAddition { .. }, - .. - } - ))); - - self.process_document_addition_batch(batch).await - } else { - if let Some(task) = batch.tasks.first_mut() { - task.events - .push(TaskEvent::Processing(OffsetDateTime::now_utc())); - - match self.process_task(task).await { - Ok(success) => { - task.events.push(TaskEvent::Succeded { - result: success, - timestamp: OffsetDateTime::now_utc(), - }); - } - Err(err) => task.events.push(TaskEvent::Failed { - error: err.into(), - timestamp: OffsetDateTime::now_utc(), - }), - } + impl IndexResolver { + pub fn load_dump( + src: impl AsRef, + dst: impl AsRef, + index_db_size: usize, + env: Arc, + indexer_opts: &IndexerOpts, + ) -> anyhow::Result<()> { + HeedMetaStore::load_dump(&src, env)?; + let indexes_path = src.as_ref().join("indexes"); + let indexes = indexes_path.read_dir()?; + let indexer_config = IndexerConfig::try_from(indexer_opts)?; + for index in indexes { + Index::load_dump(&index?.path(), &dst, index_db_size, &indexer_config)?; } - batch + + Ok(()) } } - async fn process_job(&self, job: Job) { - self.process_job(job).await; - } - - async fn finish(&self, batch: &Batch) { - for task in &batch.tasks { - if let Some(content_uuid) = task.get_content_uuid() { - if let Err(e) = self.file_store.delete(content_uuid).await { - log::error!("error deleting update file: {}", e); - } - } - } - } -} - -pub struct IndexResolver { - index_uuid_store: U, - index_store: I, - file_store: UpdateFileStore, -} - -impl IndexResolver { - pub fn load_dump( - src: impl AsRef, - dst: impl AsRef, - index_db_size: usize, - env: Arc, - indexer_opts: &IndexerOpts, - ) -> anyhow::Result<()> { - HeedMetaStore::load_dump(&src, env)?; - let indexes_path = src.as_ref().join("indexes"); - let indexes = indexes_path.read_dir()?; - let indexer_config = IndexerConfig::try_from(indexer_opts)?; - for index in indexes { - Index::load_dump(&index?.path(), &dst, index_db_size, &indexer_config)?; - } - - Ok(()) - } -} - -impl IndexResolver -where - U: IndexMetaStore, - I: IndexStore, -{ - pub fn new(index_uuid_store: U, index_store: I, file_store: UpdateFileStore) -> Self { - Self { - index_uuid_store, - index_store, - file_store, - } - } - - async fn process_document_addition_batch(&self, mut batch: Batch) -> Batch { - fn get_content_uuid(task: &Task) -> Uuid { - match task { - Task { - content: TaskContent::DocumentAddition { content_uuid, .. }, - .. - } => *content_uuid, - _ => panic!("unexpected task in the document addition batch"), + impl IndexResolver + where + U: IndexMetaStore, + I: IndexStore, + { + pub fn new(index_uuid_store: U, index_store: I, file_store: UpdateFileStore) -> Self { + Self { + index_uuid_store, + index_store, + file_store, } } - let content_uuids = batch.tasks.iter().map(get_content_uuid).collect::>(); - - match batch.tasks.first() { - Some(Task { - index_uid, - id, - content: - TaskContent::DocumentAddition { - merge_strategy, - primary_key, - allow_index_creation, + pub async fn process_document_addition_batch(&self, tasks: &mut [Task]) { + fn get_content_uuid(task: &Task) -> Uuid { + match task { + Task { + content: TaskContent::DocumentAddition { content_uuid, .. }, .. - }, - .. - }) => { - let primary_key = primary_key.clone(); - let method = *merge_strategy; + } => *content_uuid, + _ => panic!("unexpected task in the document addition batch"), + } + } - let index = if *allow_index_creation { - self.get_or_create_index(index_uid.clone(), *id).await - } else { - self.get_index(index_uid.as_str().to_string()).await - }; + let content_uuids = tasks.iter().map(get_content_uuid).collect::>(); - // If the index doesn't exist and we are not allowed to create it with the first - // task, we must fails the whole batch. - let now = OffsetDateTime::now_utc(); - let index = match index { - Ok(index) => index, - Err(e) => { - let error = ResponseError::from(e); - for task in batch.tasks.iter_mut() { - task.events.push(TaskEvent::Failed { - error: error.clone(), - timestamp: now, - }); - } - return batch; - } - }; - - let file_store = self.file_store.clone(); - let result = spawn_blocking(move || { - index.update_documents( - method, - primary_key, - file_store, - content_uuids.into_iter(), - ) - }) - .await; - - let event = match result { - Ok(Ok(result)) => TaskEvent::Succeded { - timestamp: OffsetDateTime::now_utc(), - result: TaskResult::DocumentAddition { - indexed_documents: result.indexed_documents, + match tasks.first() { + Some(Task { + id, + content: + TaskContent::DocumentAddition { + merge_strategy, + primary_key, + allow_index_creation, + index_uid, + .. }, - }, - Ok(Err(e)) => TaskEvent::Failed { - timestamp: OffsetDateTime::now_utc(), - error: e.into(), - }, - Err(e) => TaskEvent::Failed { - timestamp: OffsetDateTime::now_utc(), - error: IndexResolverError::from(e).into(), - }, - }; - - for task in batch.tasks.iter_mut() { - task.events.push(event.clone()); - } - - batch - } - _ => panic!("invalid batch!"), - } - } - - async fn process_task(&self, task: &Task) -> Result { - let index_uid = task.index_uid.clone(); - match &task.content { - TaskContent::DocumentAddition { .. } => panic!("updates should be handled by batch"), - TaskContent::DocumentDeletion(DocumentDeletion::Ids(ids)) => { - let ids = ids.clone(); - let index = self.get_index(index_uid.into_inner()).await?; - - let DocumentDeletionResult { - deleted_documents, .. - } = spawn_blocking(move || index.delete_documents(&ids)).await??; - - Ok(TaskResult::DocumentDeletion { deleted_documents }) - } - TaskContent::DocumentDeletion(DocumentDeletion::Clear) => { - let index = self.get_index(index_uid.into_inner()).await?; - let deleted_documents = spawn_blocking(move || -> IndexResult { - let number_documents = index.stats()?.number_of_documents; - index.clear_documents()?; - Ok(number_documents) - }) - .await??; - - Ok(TaskResult::ClearAll { deleted_documents }) - } - TaskContent::SettingsUpdate { - settings, - is_deletion, - allow_index_creation, - } => { - let index = if *is_deletion || !*allow_index_creation { - self.get_index(index_uid.into_inner()).await? - } else { - self.get_or_create_index(index_uid, task.id).await? - }; - - let settings = settings.clone(); - spawn_blocking(move || index.update_settings(&settings.check())).await??; - - Ok(TaskResult::Other) - } - TaskContent::IndexDeletion => { - let index = self.delete_index(index_uid.into_inner()).await?; - - let deleted_documents = spawn_blocking(move || -> IndexResult { - Ok(index.stats()?.number_of_documents) - }) - .await??; - - Ok(TaskResult::ClearAll { deleted_documents }) - } - TaskContent::IndexCreation { primary_key } => { - let index = self.create_index(index_uid, task.id).await?; - - if let Some(primary_key) = primary_key { + .. + }) => { let primary_key = primary_key.clone(); - spawn_blocking(move || index.update_primary_key(primary_key)).await??; - } + let method = *merge_strategy; - Ok(TaskResult::Other) - } - TaskContent::IndexUpdate { primary_key } => { - let index = self.get_index(index_uid.into_inner()).await?; + let index = if *allow_index_creation { + self.get_or_create_index(index_uid.clone(), *id).await + } else { + self.get_index(index_uid.as_str().to_string()).await + }; - if let Some(primary_key) = primary_key { - let primary_key = primary_key.clone(); - spawn_blocking(move || index.update_primary_key(primary_key)).await??; - } - - Ok(TaskResult::Other) - } - } - } - - async fn process_job(&self, job: Job) { - match job { - Job::Dump { ret, path } => { - log::trace!("The Dump task is getting executed"); - - let (sender, receiver) = oneshot::channel(); - if ret.send(self.dump(path).await.map(|_| sender)).is_err() { - log::error!("The dump actor died."); - } - - // wait until the dump has finished performing. - let _ = receiver.await; - } - Job::Empty => log::error!("Tried to process an empty task."), - Job::Snapshot(job) => { - if let Err(e) = job.run().await { - log::error!("Error performing snapshot: {}", e); - } - } - } - } - - pub async fn dump(&self, path: impl AsRef) -> Result<()> { - for (_, index) in self.list().await? { - index.dump(&path)?; - } - self.index_uuid_store.dump(path.as_ref().to_owned()).await?; - Ok(()) - } - - async fn create_index(&self, uid: IndexUid, creation_task_id: TaskId) -> Result { - match self.index_uuid_store.get(uid.into_inner()).await? { - (uid, Some(_)) => Err(IndexResolverError::IndexAlreadyExists(uid)), - (uid, None) => { - let uuid = Uuid::new_v4(); - let index = self.index_store.create(uuid).await?; - match self - .index_uuid_store - .insert( - uid, - IndexMeta { - uuid, - creation_task_id, - }, - ) - .await - { - Err(e) => { - match self.index_store.delete(uuid).await { - Ok(Some(index)) => { - index.close(); + // If the index doesn't exist and we are not allowed to create it with the first + // task, we must fails the whole batch. + let now = OffsetDateTime::now_utc(); + let index = match index { + Ok(index) => index, + Err(e) => { + let error = ResponseError::from(e); + for task in tasks.iter_mut() { + task.events.push(TaskEvent::Failed { + error: error.clone(), + timestamp: now, + }); } - Ok(None) => (), - Err(e) => log::error!("Error while deleting index: {:?}", e), + + return; } - Err(e) + }; + + let file_store = self.file_store.clone(); + let result = spawn_blocking(move || { + index.update_documents( + method, + primary_key, + file_store, + content_uuids.into_iter(), + ) + }) + .await; + + let event = match result { + Ok(Ok(result)) => TaskEvent::Succeeded { + timestamp: OffsetDateTime::now_utc(), + result: TaskResult::DocumentAddition { + indexed_documents: result.indexed_documents, + }, + }, + Ok(Err(e)) => TaskEvent::Failed { + timestamp: OffsetDateTime::now_utc(), + error: e.into(), + }, + Err(e) => TaskEvent::Failed { + timestamp: OffsetDateTime::now_utc(), + error: IndexResolverError::from(e).into(), + }, + }; + + for task in tasks.iter_mut() { + task.events.push(event.clone()); } - Ok(()) => Ok(index), } + _ => panic!("invalid batch!"), } } - } - /// Get or create an index with name `uid`. - pub async fn get_or_create_index(&self, uid: IndexUid, task_id: TaskId) -> Result { - match self.create_index(uid, task_id).await { - Ok(index) => Ok(index), - Err(IndexResolverError::IndexAlreadyExists(uid)) => self.get_index(uid).await, - Err(e) => Err(e), + pub async fn delete_content_file(&self, content_uuid: Uuid) -> Result<()> { + self.file_store.delete(content_uuid).await?; + Ok(()) } - } - pub async fn list(&self) -> Result> { - let uuids = self.index_uuid_store.list().await?; - let mut indexes = Vec::new(); - for (name, IndexMeta { uuid, .. }) in uuids { - match self.index_store.get(uuid).await? { - Some(index) => indexes.push((name, index)), - None => { - // we found an unexisting index, we remove it from the uuid store - let _ = self.index_uuid_store.delete(name).await; + async fn process_task_inner(&self, task: &Task) -> Result { + match &task.content { + TaskContent::DocumentAddition { .. } => { + panic!("updates should be handled by batch") + } + TaskContent::DocumentDeletion { + deletion: DocumentDeletion::Ids(ids), + index_uid, + } => { + let ids = ids.clone(); + let index = self.get_index(index_uid.clone().into_inner()).await?; + + let DocumentDeletionResult { + deleted_documents, .. + } = spawn_blocking(move || index.delete_documents(&ids)).await??; + + Ok(TaskResult::DocumentDeletion { deleted_documents }) + } + TaskContent::DocumentDeletion { + deletion: DocumentDeletion::Clear, + index_uid, + } => { + let index = self.get_index(index_uid.clone().into_inner()).await?; + let deleted_documents = spawn_blocking(move || -> IndexResult { + let number_documents = index.stats()?.number_of_documents; + index.clear_documents()?; + Ok(number_documents) + }) + .await??; + + Ok(TaskResult::ClearAll { deleted_documents }) + } + TaskContent::SettingsUpdate { + settings, + is_deletion, + allow_index_creation, + index_uid, + } => { + let index = if *is_deletion || !*allow_index_creation { + self.get_index(index_uid.clone().into_inner()).await? + } else { + self.get_or_create_index(index_uid.clone(), task.id).await? + }; + + let settings = settings.clone(); + spawn_blocking(move || index.update_settings(&settings.check())).await??; + + Ok(TaskResult::Other) + } + TaskContent::IndexDeletion { index_uid } => { + let index = self.delete_index(index_uid.clone().into_inner()).await?; + + let deleted_documents = spawn_blocking(move || -> IndexResult { + Ok(index.stats()?.number_of_documents) + }) + .await??; + + Ok(TaskResult::ClearAll { deleted_documents }) + } + TaskContent::IndexCreation { + primary_key, + index_uid, + } => { + let index = self.create_index(index_uid.clone(), task.id).await?; + + if let Some(primary_key) = primary_key { + let primary_key = primary_key.clone(); + spawn_blocking(move || index.update_primary_key(primary_key)).await??; + } + + Ok(TaskResult::Other) + } + TaskContent::IndexUpdate { + primary_key, + index_uid, + } => { + let index = self.get_index(index_uid.clone().into_inner()).await?; + + if let Some(primary_key) = primary_key { + let primary_key = primary_key.clone(); + spawn_blocking(move || index.update_primary_key(primary_key)).await??; + } + + Ok(TaskResult::Other) + } + _ => unreachable!("Invalid task for index resolver"), + } + } + + pub async fn process_task(&self, task: &mut Task) { + match self.process_task_inner(task).await { + Ok(res) => task.events.push(TaskEvent::succeeded(res)), + Err(e) => task.events.push(TaskEvent::failed(e)), + } + } + + pub async fn dump(&self, path: impl AsRef) -> Result<()> { + for (_, index) in self.list().await? { + index.dump(&path)?; + } + self.index_uuid_store.dump(path.as_ref().to_owned()).await?; + Ok(()) + } + + async fn create_index(&self, uid: IndexUid, creation_task_id: TaskId) -> Result { + match self.index_uuid_store.get(uid.into_inner()).await? { + (uid, Some(_)) => Err(IndexResolverError::IndexAlreadyExists(uid)), + (uid, None) => { + let uuid = Uuid::new_v4(); + let index = self.index_store.create(uuid).await?; + match self + .index_uuid_store + .insert( + uid, + IndexMeta { + uuid, + creation_task_id, + }, + ) + .await + { + Err(e) => { + match self.index_store.delete(uuid).await { + Ok(Some(index)) => { + index.close(); + } + Ok(None) => (), + Err(e) => log::error!("Error while deleting index: {:?}", e), + } + Err(e) + } + Ok(()) => Ok(index), + } } } } - Ok(indexes) - } - - pub async fn delete_index(&self, uid: String) -> Result { - match self.index_uuid_store.delete(uid.clone()).await? { - Some(IndexMeta { uuid, .. }) => match self.index_store.delete(uuid).await? { - Some(index) => { - index.clone().close(); - Ok(index) - } - None => Err(IndexResolverError::UnexistingIndex(uid)), - }, - None => Err(IndexResolverError::UnexistingIndex(uid)), + /// Get or create an index with name `uid`. + pub async fn get_or_create_index(&self, uid: IndexUid, task_id: TaskId) -> Result { + match self.create_index(uid, task_id).await { + Ok(index) => Ok(index), + Err(IndexResolverError::IndexAlreadyExists(uid)) => self.get_index(uid).await, + Err(e) => Err(e), + } } - } - pub async fn get_index(&self, uid: String) -> Result { - match self.index_uuid_store.get(uid).await? { - (name, Some(IndexMeta { uuid, .. })) => { + pub async fn list(&self) -> Result> { + let uuids = self.index_uuid_store.list().await?; + let mut indexes = Vec::new(); + for (name, IndexMeta { uuid, .. }) in uuids { match self.index_store.get(uuid).await? { - Some(index) => Ok(index), + Some(index) => indexes.push((name, index)), None => { - // For some reason we got a uuid to an unexisting index, we return an error, - // and remove the uuid from the uuid store. - let _ = self.index_uuid_store.delete(name.clone()).await; - Err(IndexResolverError::UnexistingIndex(name)) + // we found an unexisting index, we remove it from the uuid store + let _ = self.index_uuid_store.delete(name).await; } } } - (name, _) => Err(IndexResolverError::UnexistingIndex(name)), - } - } - pub async fn get_index_creation_task_id(&self, index_uid: String) -> Result { - let (uid, meta) = self.index_uuid_store.get(index_uid).await?; - meta.map( - |IndexMeta { - creation_task_id, .. - }| creation_task_id, - ) - .ok_or(IndexResolverError::UnexistingIndex(uid)) + Ok(indexes) + } + + pub async fn delete_index(&self, uid: String) -> Result { + match self.index_uuid_store.delete(uid.clone()).await? { + Some(IndexMeta { uuid, .. }) => match self.index_store.delete(uuid).await? { + Some(index) => { + index.clone().close(); + Ok(index) + } + None => Err(IndexResolverError::UnexistingIndex(uid)), + }, + None => Err(IndexResolverError::UnexistingIndex(uid)), + } + } + + pub async fn get_index(&self, uid: String) -> Result { + match self.index_uuid_store.get(uid).await? { + (name, Some(IndexMeta { uuid, .. })) => { + match self.index_store.get(uuid).await? { + Some(index) => Ok(index), + None => { + // For some reason we got a uuid to an unexisting index, we return an error, + // and remove the uuid from the uuid store. + let _ = self.index_uuid_store.delete(name.clone()).await; + Err(IndexResolverError::UnexistingIndex(name)) + } + } + } + (name, _) => Err(IndexResolverError::UnexistingIndex(name)), + } + } + + pub async fn get_index_creation_task_id(&self, index_uid: String) -> Result { + let (uid, meta) = self.index_uuid_store.get(index_uid).await?; + meta.map( + |IndexMeta { + creation_task_id, .. + }| creation_task_id, + ) + .ok_or(IndexResolverError::UnexistingIndex(uid)) + } } } #[cfg(test)] mod test { - use std::{collections::BTreeMap, vec::IntoIter}; + use crate::index::IndexStats; + use super::index_store::MockIndexStore; + use super::meta_store::MockIndexMetaStore; use super::*; use futures::future::ok; - use milli::update::{DocumentAdditionResult, IndexDocumentsMethod}; + use milli::FieldDistribution; use nelson::Mocker; - use proptest::prelude::*; - use crate::index::{ - error::{IndexError, Result as IndexResult}, - Checked, IndexMeta, IndexStats, Settings, - }; - use index_store::MockIndexStore; - use meta_store::MockIndexMetaStore; + pub enum MockIndexResolver { + Real(super::real::IndexResolver), + Mock(Mocker), + } - proptest! { - #[test] - fn test_process_task( - task in any::(), - index_exists in any::(), - index_op_fails in any::(), - any_int in any::(), - ) { - actix_rt::System::new().block_on(async move { - let uuid = Uuid::new_v4(); - let mut index_store = MockIndexStore::new(); - - let mocker = Mocker::default(); - - // Return arbitrary data from index call. - match &task.content { - TaskContent::DocumentAddition{primary_key, ..} => { - let result = move || if !index_op_fails { - Ok(DocumentAdditionResult { indexed_documents: any_int, number_of_documents: any_int }) - } else { - // return this error because it's easy to generate... - Err(IndexError::DocumentNotFound("a doc".into())) - }; - if primary_key.is_some() { - mocker.when::>("update_primary_key") - .then(move |_| Ok(IndexMeta{ created_at: OffsetDateTime::now_utc(), updated_at: OffsetDateTime::now_utc(), primary_key: None })); - } - mocker.when::<(IndexDocumentsMethod, Option, UpdateFileStore, IntoIter), IndexResult>("update_documents") - .then(move |(_, _, _, _)| result()); - } - TaskContent::SettingsUpdate{..} => { - let result = move || if !index_op_fails { - Ok(()) - } else { - // return this error because it's easy to generate... - Err(IndexError::DocumentNotFound("a doc".into())) - }; - mocker.when::<&Settings, IndexResult<()>>("update_settings") - .then(move |_| result()); - } - TaskContent::DocumentDeletion(DocumentDeletion::Ids(_ids)) => { - let result = move || if !index_op_fails { - Ok(DocumentDeletionResult { deleted_documents: any_int as u64, remaining_documents: any_int as u64 }) - } else { - // return this error because it's easy to generate... - Err(IndexError::DocumentNotFound("a doc".into())) - }; - - mocker.when::<&[String], IndexResult>("delete_documents") - .then(move |_| result()); - }, - TaskContent::DocumentDeletion(DocumentDeletion::Clear) => { - let result = move || if !index_op_fails { - Ok(()) - } else { - // return this error because it's easy to generate... - Err(IndexError::DocumentNotFound("a doc".into())) - }; - mocker.when::<(), IndexResult<()>>("clear_documents") - .then(move |_| result()); - }, - TaskContent::IndexDeletion => { - mocker.when::<(), ()>("close") - .times(index_exists as usize) - .then(move |_| ()); - } - TaskContent::IndexUpdate { primary_key } - | TaskContent::IndexCreation { primary_key } => { - if primary_key.is_some() { - let result = move || if !index_op_fails { - Ok(IndexMeta{ created_at: OffsetDateTime::now_utc(), updated_at: OffsetDateTime::now_utc(), primary_key: None }) - } else { - // return this error because it's easy to generate... - Err(IndexError::DocumentNotFound("a doc".into())) - }; - mocker.when::>("update_primary_key") - .then(move |_| result()); - } - } - } - - mocker.when::<(), IndexResult>("stats") - .then(|()| Ok(IndexStats { size: 0, number_of_documents: 0, is_indexing: Some(false), field_distribution: BTreeMap::new() })); - - let index = Index::mock(mocker); - - match &task.content { - // an unexisting index should trigger an index creation in the folllowing cases: - TaskContent::DocumentAddition { allow_index_creation: true, .. } - | TaskContent::SettingsUpdate { allow_index_creation: true, is_deletion: false, .. } - | TaskContent::IndexCreation { .. } if !index_exists => { - index_store - .expect_create() - .once() - .withf(move |&found| !index_exists || found == uuid) - .returning(move |_| Box::pin(ok(index.clone()))); - }, - TaskContent::IndexDeletion => { - index_store - .expect_delete() - // this is called only if the index.exists - .times(index_exists as usize) - .withf(move |&found| !index_exists || found == uuid) - .returning(move |_| Box::pin(ok(Some(index.clone())))); - } - // if index already exists, create index will return an error - TaskContent::IndexCreation { .. } if index_exists => (), - // The index exists and get should be called - _ if index_exists => { - index_store - .expect_get() - .once() - .withf(move |&found| found == uuid) - .returning(move |_| Box::pin(ok(Some(index.clone())))); - }, - // the index doesn't exist and shouldn't be created, the uuidstore will return an error, and get_index will never be called. - _ => (), - } - - let mut uuid_store = MockIndexMetaStore::new(); - uuid_store - .expect_get() - .returning(move |uid| { - Box::pin(ok((uid, index_exists.then(|| crate::index_resolver::meta_store::IndexMeta {uuid, creation_task_id: 0 })))) - }); - - // we sould only be creating an index if the index doesn't alredy exist - uuid_store - .expect_insert() - .withf(move |_, _| !index_exists) - .returning(|_, _| Box::pin(ok(()))); - - uuid_store - .expect_delete() - .times(matches!(task.content, TaskContent::IndexDeletion) as usize) - .returning(move |_| Box::pin(ok(index_exists.then(|| crate::index_resolver::meta_store::IndexMeta { uuid, creation_task_id: 0})))); - - let mocker = Mocker::default(); - let update_file_store = UpdateFileStore::mock(mocker); - let index_resolver = IndexResolver::new(uuid_store, index_store, update_file_store); - - let batch = Batch { id: 1, created_at: OffsetDateTime::now_utc(), tasks: vec![task.clone()] }; - let result = index_resolver.process_batch(batch).await; - - // Test for some expected output scenarios: - // Index creation and deletion cannot fail because of a failed index op, since they - // don't perform index ops. - if index_op_fails && !matches!(task.content, TaskContent::IndexDeletion | TaskContent::IndexCreation { primary_key: None } | TaskContent::IndexUpdate { primary_key: None }) - || (index_exists && matches!(task.content, TaskContent::IndexCreation { .. })) - || (!index_exists && matches!(task.content, TaskContent::IndexDeletion - | TaskContent::DocumentDeletion(_) - | TaskContent::SettingsUpdate { is_deletion: true, ..} - | TaskContent::SettingsUpdate { allow_index_creation: false, ..} - | TaskContent::DocumentAddition { allow_index_creation: false, ..} - | TaskContent::IndexUpdate { .. } )) - { - assert!(matches!(result.tasks[0].events.last().unwrap(), TaskEvent::Failed { .. }), "{:?}", result); - } else { - assert!(matches!(result.tasks[0].events.last().unwrap(), TaskEvent::Succeded { .. }), "{:?}", result); - } - }); + impl MockIndexResolver { + pub fn load_dump( + src: impl AsRef, + dst: impl AsRef, + index_db_size: usize, + env: Arc, + indexer_opts: &IndexerOpts, + ) -> anyhow::Result<()> { + super::real::IndexResolver::load_dump(src, dst, index_db_size, env, indexer_opts) } } + + impl MockIndexResolver + where + U: IndexMetaStore, + I: IndexStore, + { + pub fn new(index_uuid_store: U, index_store: I, file_store: UpdateFileStore) -> Self { + Self::Real(super::real::IndexResolver { + index_uuid_store, + index_store, + file_store, + }) + } + + pub fn mock(mocker: Mocker) -> Self { + Self::Mock(mocker) + } + + pub async fn process_document_addition_batch(&self, tasks: &mut [Task]) { + match self { + IndexResolver::Real(r) => r.process_document_addition_batch(tasks).await, + IndexResolver::Mock(m) => unsafe { + m.get("process_document_addition_batch").call(tasks) + }, + } + } + + pub async fn process_task(&self, task: &mut Task) { + match self { + IndexResolver::Real(r) => r.process_task(task).await, + IndexResolver::Mock(m) => unsafe { m.get("process_task").call(task) }, + } + } + + pub async fn dump(&self, path: impl AsRef) -> Result<()> { + match self { + IndexResolver::Real(r) => r.dump(path).await, + IndexResolver::Mock(_) => todo!(), + } + } + + /// Get or create an index with name `uid`. + pub async fn get_or_create_index(&self, uid: IndexUid, task_id: TaskId) -> Result { + match self { + IndexResolver::Real(r) => r.get_or_create_index(uid, task_id).await, + IndexResolver::Mock(_) => todo!(), + } + } + + pub async fn list(&self) -> Result> { + match self { + IndexResolver::Real(r) => r.list().await, + IndexResolver::Mock(_) => todo!(), + } + } + + pub async fn delete_index(&self, uid: String) -> Result { + match self { + IndexResolver::Real(r) => r.delete_index(uid).await, + IndexResolver::Mock(_) => todo!(), + } + } + + pub async fn get_index(&self, uid: String) -> Result { + match self { + IndexResolver::Real(r) => r.get_index(uid).await, + IndexResolver::Mock(_) => todo!(), + } + } + + pub async fn get_index_creation_task_id(&self, index_uid: String) -> Result { + match self { + IndexResolver::Real(r) => r.get_index_creation_task_id(index_uid).await, + IndexResolver::Mock(_) => todo!(), + } + } + + pub async fn delete_content_file(&self, content_uuid: Uuid) -> Result<()> { + match self { + IndexResolver::Real(r) => r.delete_content_file(content_uuid).await, + IndexResolver::Mock(m) => unsafe { + m.get("delete_content_file").call(content_uuid) + }, + } + } + } + + #[actix_rt::test] + async fn test_remove_unknown_index() { + let mut meta_store = MockIndexMetaStore::new(); + meta_store + .expect_delete() + .once() + .returning(|_| Box::pin(ok(None))); + + let index_store = MockIndexStore::new(); + + let mocker = Mocker::default(); + let file_store = UpdateFileStore::mock(mocker); + + let index_resolver = IndexResolver::new(meta_store, index_store, file_store); + + let mut task = Task { + id: 1, + content: TaskContent::IndexDeletion { + index_uid: IndexUid::new_unchecked("test"), + }, + events: Vec::new(), + }; + + index_resolver.process_task(&mut task).await; + + assert!(matches!(task.events[0], TaskEvent::Failed { .. })); + } + + #[actix_rt::test] + async fn test_remove_index() { + let mut meta_store = MockIndexMetaStore::new(); + meta_store.expect_delete().once().returning(|_| { + Box::pin(ok(Some(IndexMeta { + uuid: Uuid::new_v4(), + creation_task_id: 1, + }))) + }); + + let mut index_store = MockIndexStore::new(); + index_store.expect_delete().once().returning(|_| { + let mocker = Mocker::default(); + mocker.when::<(), ()>("close").then(|_| ()); + mocker + .when::<(), IndexResult>("stats") + .then(|_| { + Ok(IndexStats { + size: 10, + number_of_documents: 10, + is_indexing: None, + field_distribution: FieldDistribution::default(), + }) + }); + Box::pin(ok(Some(Index::mock(mocker)))) + }); + + let mocker = Mocker::default(); + let file_store = UpdateFileStore::mock(mocker); + + let index_resolver = IndexResolver::new(meta_store, index_store, file_store); + + let mut task = Task { + id: 1, + content: TaskContent::IndexDeletion { + index_uid: IndexUid::new_unchecked("test"), + }, + events: Vec::new(), + }; + + index_resolver.process_task(&mut task).await; + + assert!(matches!(task.events[0], TaskEvent::Succeeded { .. })); + } + + #[actix_rt::test] + async fn test_delete_documents() { + let mut meta_store = MockIndexMetaStore::new(); + meta_store.expect_get().once().returning(|_| { + Box::pin(ok(( + "test".to_string(), + Some(IndexMeta { + uuid: Uuid::new_v4(), + creation_task_id: 1, + }), + ))) + }); + + let mut index_store = MockIndexStore::new(); + index_store.expect_get().once().returning(|_| { + let mocker = Mocker::default(); + mocker + .when::<(), IndexResult<()>>("clear_documents") + .once() + .then(|_| Ok(())); + mocker + .when::<(), IndexResult>("stats") + .once() + .then(|_| { + Ok(IndexStats { + size: 10, + number_of_documents: 10, + is_indexing: None, + field_distribution: FieldDistribution::default(), + }) + }); + Box::pin(ok(Some(Index::mock(mocker)))) + }); + + let mocker = Mocker::default(); + let file_store = UpdateFileStore::mock(mocker); + + let index_resolver = IndexResolver::new(meta_store, index_store, file_store); + + let mut task = Task { + id: 1, + content: TaskContent::DocumentDeletion { + deletion: DocumentDeletion::Clear, + index_uid: IndexUid::new_unchecked("test"), + }, + events: Vec::new(), + }; + + index_resolver.process_task(&mut task).await; + + assert!(matches!(task.events[0], TaskEvent::Succeeded { .. })); + } + + #[actix_rt::test] + async fn test_index_update() { + let mut meta_store = MockIndexMetaStore::new(); + meta_store.expect_get().once().returning(|_| { + Box::pin(ok(( + "test".to_string(), + Some(IndexMeta { + uuid: Uuid::new_v4(), + creation_task_id: 1, + }), + ))) + }); + + let mut index_store = MockIndexStore::new(); + index_store.expect_get().once().returning(|_| { + let mocker = Mocker::default(); + + mocker + .when::>("update_primary_key") + .once() + .then(|_| { + Ok(crate::index::IndexMeta { + created_at: OffsetDateTime::now_utc(), + updated_at: OffsetDateTime::now_utc(), + primary_key: Some("key".to_string()), + }) + }); + Box::pin(ok(Some(Index::mock(mocker)))) + }); + + let mocker = Mocker::default(); + let file_store = UpdateFileStore::mock(mocker); + + let index_resolver = IndexResolver::new(meta_store, index_store, file_store); + + let mut task = Task { + id: 1, + content: TaskContent::IndexUpdate { + primary_key: Some("key".to_string()), + index_uid: IndexUid::new_unchecked("test"), + }, + events: Vec::new(), + }; + + index_resolver.process_task(&mut task).await; + + assert!(matches!(task.events[0], TaskEvent::Succeeded { .. })); + } } diff --git a/meilisearch-lib/src/lib.rs b/meilisearch-lib/src/lib.rs index 1161340ba..70fd2ba51 100644 --- a/meilisearch-lib/src/lib.rs +++ b/meilisearch-lib/src/lib.rs @@ -3,6 +3,7 @@ pub mod error; pub mod options; mod analytics; +mod dump; pub mod index; pub mod index_controller; mod index_resolver; @@ -19,23 +20,6 @@ pub use milli::heed; mod compression; pub mod document_formats; -use walkdir::WalkDir; - -pub trait EnvSizer { - fn size(&self) -> u64; -} - -impl EnvSizer for milli::heed::Env { - fn size(&self) -> u64 { - WalkDir::new(self.path()) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| entry.metadata().ok()) - .filter(|metadata| metadata.is_file()) - .fold(0, |acc, m| acc + m.len()) - } -} - /// Check if a db is empty. It does not provide any information on the /// validity of the data in it. /// We consider a database as non empty when it's a non empty directory. diff --git a/meilisearch-lib/src/snapshot.rs b/meilisearch-lib/src/snapshot.rs index 6c27ad2f0..da4907939 100644 --- a/meilisearch-lib/src/snapshot.rs +++ b/meilisearch-lib/src/snapshot.rs @@ -7,6 +7,7 @@ use anyhow::bail; use fs_extra::dir::{self, CopyOptions}; use log::{info, trace}; use meilisearch_auth::open_auth_store_env; +use milli::heed::CompactionOption; use tokio::sync::RwLock; use tokio::time::sleep; use walkdir::WalkDir; @@ -14,7 +15,6 @@ use walkdir::WalkDir; use crate::compression::from_tar_gz; use crate::index_controller::open_meta_env; use crate::index_controller::versioning::VERSION_FILE_NAME; -use crate::tasks::task::Job; use crate::tasks::Scheduler; pub struct SnapshotService { @@ -39,8 +39,7 @@ impl SnapshotService { meta_env_size: self.meta_env_size, index_size: self.index_size, }; - let job = Job::Snapshot(snapshot_job); - self.scheduler.write().await.schedule_job(job).await; + self.scheduler.write().await.schedule_snapshot(snapshot_job); sleep(self.snapshot_period).await; } } @@ -183,9 +182,7 @@ impl SnapshotJob { let mut options = milli::heed::EnvOpenOptions::new(); options.map_size(self.index_size); let index = milli::Index::new(options, entry.path())?; - index - .env - .copy_to_path(dst, milli::heed::CompactionOption::Enabled)?; + index.copy_to_path(dst, CompactionOption::Enabled)?; } Ok(()) diff --git a/meilisearch-lib/src/tasks/batch.rs b/meilisearch-lib/src/tasks/batch.rs index 4a8cf7907..7173ecd33 100644 --- a/meilisearch-lib/src/tasks/batch.rs +++ b/meilisearch-lib/src/tasks/batch.rs @@ -1,22 +1,75 @@ use time::OffsetDateTime; -use super::task::Task; +use crate::snapshot::SnapshotJob; -pub type BatchId = u64; +use super::task::{Task, TaskEvent}; + +pub type BatchId = u32; + +#[derive(Debug)] +pub enum BatchContent { + DocumentsAdditionBatch(Vec), + IndexUpdate(Task), + Dump(Task), + Snapshot(SnapshotJob), + // Symbolizes a empty batch. This can occur when we were woken, but there wasn't any work to do. + Empty, +} + +impl BatchContent { + pub fn first(&self) -> Option<&Task> { + match self { + BatchContent::DocumentsAdditionBatch(ts) => ts.first(), + BatchContent::Dump(t) | BatchContent::IndexUpdate(t) => Some(t), + BatchContent::Snapshot(_) | BatchContent::Empty => None, + } + } + + pub fn push_event(&mut self, event: TaskEvent) { + match self { + BatchContent::DocumentsAdditionBatch(ts) => { + ts.iter_mut().for_each(|t| t.events.push(event.clone())) + } + BatchContent::IndexUpdate(t) | BatchContent::Dump(t) => t.events.push(event), + BatchContent::Snapshot(_) | BatchContent::Empty => (), + } + } +} #[derive(Debug)] pub struct Batch { - pub id: BatchId, + // Only batches that contains a persistant tasks are given an id. Snapshot batches don't have + // an id. + pub id: Option, pub created_at: OffsetDateTime, - pub tasks: Vec, + pub content: BatchContent, } impl Batch { + pub fn new(id: Option, content: BatchContent) -> Self { + Self { + id, + created_at: OffsetDateTime::now_utc(), + content, + } + } pub fn len(&self) -> usize { - self.tasks.len() + match self.content { + BatchContent::DocumentsAdditionBatch(ref ts) => ts.len(), + BatchContent::IndexUpdate(_) | BatchContent::Dump(_) | BatchContent::Snapshot(_) => 1, + BatchContent::Empty => 0, + } } pub fn is_empty(&self) -> bool { - self.tasks.is_empty() + self.len() == 0 + } + + pub fn empty() -> Self { + Self { + id: None, + created_at: OffsetDateTime::now_utc(), + content: BatchContent::Empty, + } } } diff --git a/meilisearch-lib/src/tasks/error.rs b/meilisearch-lib/src/tasks/error.rs index d849b4c10..75fd7a591 100644 --- a/meilisearch-lib/src/tasks/error.rs +++ b/meilisearch-lib/src/tasks/error.rs @@ -1,4 +1,5 @@ -use meilisearch_error::{internal_error, Code, ErrorCode}; +use meilisearch_types::error::{Code, ErrorCode}; +use meilisearch_types::internal_error; use tokio::task::JoinError; use crate::update_file_store::UpdateFileStoreError; diff --git a/meilisearch-lib/src/tasks/handlers/dump_handler.rs b/meilisearch-lib/src/tasks/handlers/dump_handler.rs new file mode 100644 index 000000000..c0833e4c7 --- /dev/null +++ b/meilisearch-lib/src/tasks/handlers/dump_handler.rs @@ -0,0 +1,132 @@ +use crate::dump::DumpHandler; +use crate::index_resolver::index_store::IndexStore; +use crate::index_resolver::meta_store::IndexMetaStore; +use crate::tasks::batch::{Batch, BatchContent}; +use crate::tasks::task::{Task, TaskContent, TaskEvent, TaskResult}; +use crate::tasks::BatchHandler; + +#[async_trait::async_trait] +impl BatchHandler for DumpHandler +where + U: IndexMetaStore + Sync + Send + 'static, + I: IndexStore + Sync + Send + 'static, +{ + fn accept(&self, batch: &Batch) -> bool { + matches!(batch.content, BatchContent::Dump { .. }) + } + + async fn process_batch(&self, mut batch: Batch) -> Batch { + match &batch.content { + BatchContent::Dump(Task { + content: TaskContent::Dump { uid }, + .. + }) => { + match self.run(uid.clone()).await { + Ok(_) => { + batch + .content + .push_event(TaskEvent::succeeded(TaskResult::Other)); + } + Err(e) => batch.content.push_event(TaskEvent::failed(e)), + } + batch + } + _ => unreachable!("invalid batch content for dump"), + } + } + + async fn finish(&self, _: &Batch) {} +} + +#[cfg(test)] +mod test { + use crate::dump::error::{DumpError, Result as DumpResult}; + use crate::index_resolver::{index_store::MockIndexStore, meta_store::MockIndexMetaStore}; + use crate::tasks::handlers::test::task_to_batch; + + use super::*; + + use nelson::Mocker; + use proptest::prelude::*; + + proptest! { + #[test] + fn finish_does_nothing( + task in any::(), + ) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let handle = rt.spawn(async { + let batch = task_to_batch(task); + + let mocker = Mocker::default(); + let dump_handler = DumpHandler::::mock(mocker); + + dump_handler.finish(&batch).await; + }); + + rt.block_on(handle).unwrap(); + } + + #[test] + fn test_handle_dump_success( + task in any::(), + ) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let handle = rt.spawn(async { + let batch = task_to_batch(task); + let should_accept = matches!(batch.content, BatchContent::Dump { .. }); + + let mocker = Mocker::default(); + if should_accept { + mocker.when::>("run") + .once() + .then(|_| Ok(())); + } + + let dump_handler = DumpHandler::::mock(mocker); + + let accept = dump_handler.accept(&batch); + assert_eq!(accept, should_accept); + + if accept { + let batch = dump_handler.process_batch(batch).await; + let last_event = batch.content.first().unwrap().events.last().unwrap(); + assert!(matches!(last_event, TaskEvent::Succeeded { .. })); + } + }); + + rt.block_on(handle).unwrap(); + } + + #[test] + fn test_handle_dump_error( + task in any::(), + ) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let handle = rt.spawn(async { + let batch = task_to_batch(task); + let should_accept = matches!(batch.content, BatchContent::Dump { .. }); + + let mocker = Mocker::default(); + if should_accept { + mocker.when::>("run") + .once() + .then(|_| Err(DumpError::Internal("error".into()))); + } + + let dump_handler = DumpHandler::::mock(mocker); + + let accept = dump_handler.accept(&batch); + assert_eq!(accept, should_accept); + + if accept { + let batch = dump_handler.process_batch(batch).await; + let last_event = batch.content.first().unwrap().events.last().unwrap(); + assert!(matches!(last_event, TaskEvent::Failed { .. })); + } + }); + + rt.block_on(handle).unwrap(); + } + } +} diff --git a/meilisearch-lib/src/tasks/handlers/empty_handler.rs b/meilisearch-lib/src/tasks/handlers/empty_handler.rs new file mode 100644 index 000000000..d800e1965 --- /dev/null +++ b/meilisearch-lib/src/tasks/handlers/empty_handler.rs @@ -0,0 +1,18 @@ +use crate::tasks::batch::{Batch, BatchContent}; +use crate::tasks::BatchHandler; + +/// A sink handler for empty tasks. +pub struct EmptyBatchHandler; + +#[async_trait::async_trait] +impl BatchHandler for EmptyBatchHandler { + fn accept(&self, batch: &Batch) -> bool { + matches!(batch.content, BatchContent::Empty) + } + + async fn process_batch(&self, batch: Batch) -> Batch { + batch + } + + async fn finish(&self, _: &Batch) {} +} diff --git a/meilisearch-lib/src/tasks/handlers/index_resolver_handler.rs b/meilisearch-lib/src/tasks/handlers/index_resolver_handler.rs new file mode 100644 index 000000000..22c57e2fd --- /dev/null +++ b/meilisearch-lib/src/tasks/handlers/index_resolver_handler.rs @@ -0,0 +1,199 @@ +use crate::index_resolver::IndexResolver; +use crate::index_resolver::{index_store::IndexStore, meta_store::IndexMetaStore}; +use crate::tasks::batch::{Batch, BatchContent}; +use crate::tasks::BatchHandler; + +#[async_trait::async_trait] +impl BatchHandler for IndexResolver +where + U: IndexMetaStore + Send + Sync + 'static, + I: IndexStore + Send + Sync + 'static, +{ + fn accept(&self, batch: &Batch) -> bool { + matches!( + batch.content, + BatchContent::DocumentsAdditionBatch(_) | BatchContent::IndexUpdate(_) + ) + } + + async fn process_batch(&self, mut batch: Batch) -> Batch { + match batch.content { + BatchContent::DocumentsAdditionBatch(ref mut tasks) => { + self.process_document_addition_batch(tasks).await; + } + BatchContent::IndexUpdate(ref mut task) => { + self.process_task(task).await; + } + _ => unreachable!(), + } + + batch + } + + async fn finish(&self, batch: &Batch) { + if let BatchContent::DocumentsAdditionBatch(ref tasks) = batch.content { + for task in tasks { + if let Some(content_uuid) = task.get_content_uuid() { + if let Err(e) = self.delete_content_file(content_uuid).await { + log::error!("error deleting update file: {}", e); + } + } + } + } + } +} + +#[cfg(test)] +mod test { + use crate::index_resolver::index_store::MapIndexStore; + use crate::index_resolver::meta_store::HeedMetaStore; + use crate::index_resolver::{ + error::Result as IndexResult, index_store::MockIndexStore, meta_store::MockIndexMetaStore, + }; + use crate::tasks::{ + handlers::test::task_to_batch, + task::{Task, TaskContent}, + }; + use crate::update_file_store::{Result as FileStoreResult, UpdateFileStore}; + + use super::*; + use meilisearch_types::index_uid::IndexUid; + use milli::update::IndexDocumentsMethod; + use nelson::Mocker; + use proptest::prelude::*; + use uuid::Uuid; + + proptest! { + #[test] + fn test_accept_task( + task in any::(), + ) { + let batch = task_to_batch(task); + + let index_store = MockIndexStore::new(); + let meta_store = MockIndexMetaStore::new(); + let mocker = Mocker::default(); + let update_file_store = UpdateFileStore::mock(mocker); + let index_resolver = IndexResolver::new(meta_store, index_store, update_file_store); + + match batch.content { + BatchContent::DocumentsAdditionBatch(_) + | BatchContent::IndexUpdate(_) => assert!(index_resolver.accept(&batch)), + BatchContent::Dump(_) + | BatchContent::Snapshot(_) + | BatchContent::Empty => assert!(!index_resolver.accept(&batch)), + } + } + } + + #[actix_rt::test] + async fn finisher_called_on_document_update() { + let index_store = MockIndexStore::new(); + let meta_store = MockIndexMetaStore::new(); + let mocker = Mocker::default(); + let content_uuid = Uuid::new_v4(); + mocker + .when::>("delete") + .once() + .then(move |uuid| { + assert_eq!(uuid, content_uuid); + Ok(()) + }); + let update_file_store = UpdateFileStore::mock(mocker); + let index_resolver = IndexResolver::new(meta_store, index_store, update_file_store); + + let task = Task { + id: 1, + content: TaskContent::DocumentAddition { + content_uuid, + merge_strategy: IndexDocumentsMethod::ReplaceDocuments, + primary_key: None, + documents_count: 100, + allow_index_creation: true, + index_uid: IndexUid::new_unchecked("test"), + }, + events: Vec::new(), + }; + + let batch = task_to_batch(task); + + index_resolver.finish(&batch).await; + } + + #[actix_rt::test] + #[should_panic] + async fn panic_when_passed_unsupported_batch() { + let index_store = MockIndexStore::new(); + let meta_store = MockIndexMetaStore::new(); + let mocker = Mocker::default(); + let update_file_store = UpdateFileStore::mock(mocker); + let index_resolver = IndexResolver::new(meta_store, index_store, update_file_store); + + let task = Task { + id: 1, + content: TaskContent::Dump { + uid: String::from("hello"), + }, + events: Vec::new(), + }; + + let batch = task_to_batch(task); + + index_resolver.process_batch(batch).await; + } + + proptest! { + #[test] + fn index_document_task_deletes_update_file( + task in any::(), + ) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let handle = rt.spawn(async { + let mocker = Mocker::default(); + + if let TaskContent::DocumentAddition{ .. } = task.content { + mocker.when::>("delete_content_file").then(|_| Ok(())); + } + + let index_resolver: IndexResolver = IndexResolver::mock(mocker); + + let batch = task_to_batch(task); + + index_resolver.finish(&batch).await; + }); + + rt.block_on(handle).unwrap(); + } + + #[test] + fn test_handle_batch(task in any::()) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let handle = rt.spawn(async { + let mocker = Mocker::default(); + match task.content { + TaskContent::DocumentAddition { .. } => { + mocker.when::<&mut [Task], ()>("process_document_addition_batch").then(|_| ()); + } + TaskContent::Dump { .. } => (), + _ => { + mocker.when::<&mut Task, ()>("process_task").then(|_| ()); + } + } + let index_resolver: IndexResolver = IndexResolver::mock(mocker); + + + let batch = task_to_batch(task); + + if index_resolver.accept(&batch) { + index_resolver.process_batch(batch).await; + } + }); + + if let Err(e) = rt.block_on(handle) { + if e.is_panic() { + std::panic::resume_unwind(e.into_panic()); + } + } + } + } +} diff --git a/meilisearch-lib/src/tasks/handlers/mod.rs b/meilisearch-lib/src/tasks/handlers/mod.rs new file mode 100644 index 000000000..8f02de8b9 --- /dev/null +++ b/meilisearch-lib/src/tasks/handlers/mod.rs @@ -0,0 +1,34 @@ +pub mod dump_handler; +pub mod empty_handler; +mod index_resolver_handler; +pub mod snapshot_handler; + +#[cfg(test)] +mod test { + use time::OffsetDateTime; + + use crate::tasks::{ + batch::{Batch, BatchContent}, + task::{Task, TaskContent}, + }; + + pub fn task_to_batch(task: Task) -> Batch { + let content = match task.content { + TaskContent::DocumentAddition { .. } => { + BatchContent::DocumentsAdditionBatch(vec![task]) + } + TaskContent::DocumentDeletion { .. } + | TaskContent::SettingsUpdate { .. } + | TaskContent::IndexDeletion { .. } + | TaskContent::IndexCreation { .. } + | TaskContent::IndexUpdate { .. } => BatchContent::IndexUpdate(task), + TaskContent::Dump { .. } => BatchContent::Dump(task), + }; + + Batch { + id: Some(1), + created_at: OffsetDateTime::now_utc(), + content, + } + } +} diff --git a/meilisearch-lib/src/tasks/handlers/snapshot_handler.rs b/meilisearch-lib/src/tasks/handlers/snapshot_handler.rs new file mode 100644 index 000000000..32fe6d746 --- /dev/null +++ b/meilisearch-lib/src/tasks/handlers/snapshot_handler.rs @@ -0,0 +1,26 @@ +use crate::tasks::batch::{Batch, BatchContent}; +use crate::tasks::BatchHandler; + +pub struct SnapshotHandler; + +#[async_trait::async_trait] +impl BatchHandler for SnapshotHandler { + fn accept(&self, batch: &Batch) -> bool { + matches!(batch.content, BatchContent::Snapshot(_)) + } + + async fn process_batch(&self, batch: Batch) -> Batch { + match batch.content { + BatchContent::Snapshot(job) => { + if let Err(e) = job.run().await { + log::error!("snapshot error: {e}"); + } + } + _ => unreachable!(), + } + + Batch::empty() + } + + async fn finish(&self, _: &Batch) {} +} diff --git a/meilisearch-lib/src/tasks/mod.rs b/meilisearch-lib/src/tasks/mod.rs index b56dfaf9d..d8bc25bb7 100644 --- a/meilisearch-lib/src/tasks/mod.rs +++ b/meilisearch-lib/src/tasks/mod.rs @@ -1,5 +1,7 @@ use async_trait::async_trait; +pub use handlers::empty_handler::EmptyBatchHandler; +pub use handlers::snapshot_handler::SnapshotHandler; pub use scheduler::Scheduler; pub use task_store::TaskFilter; @@ -11,10 +13,9 @@ pub use task_store::TaskStore; use batch::Batch; use error::Result; -use self::task::Job; - pub mod batch; pub mod error; +mod handlers; mod scheduler; pub mod task; mod task_store; @@ -22,11 +23,15 @@ pub mod update_loop; #[cfg_attr(test, mockall::automock(type Error=test::DebugError;))] #[async_trait] -pub trait TaskPerformer: Sync + Send + 'static { - /// Processes the `Task` batch returning the batch with the `Task` updated. - async fn process_batch(&self, batch: Batch) -> Batch; +pub trait BatchHandler: Sync + Send + 'static { + /// return whether this handler can accept this batch + fn accept(&self, batch: &Batch) -> bool; - async fn process_job(&self, job: Job); + /// Processes the `Task` batch returning the batch with the `Task` updated. + /// + /// It is ok for this function to panic if a batch is handed that hasn't been verified by + /// `accept` beforehand. + async fn process_batch(&self, batch: Batch) -> Batch; /// `finish` is called when the result of `process` has been commited to the task store. This /// method can be used to perform cleanup after the update has been completed for example. diff --git a/meilisearch-lib/src/tasks/scheduler.rs b/meilisearch-lib/src/tasks/scheduler.rs index 0e540a646..8ce14fe8c 100644 --- a/meilisearch-lib/src/tasks/scheduler.rs +++ b/meilisearch-lib/src/tasks/scheduler.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; use std::collections::{hash_map::Entry, BinaryHeap, HashMap, VecDeque}; use std::ops::{Deref, DerefMut}; -use std::path::Path; +use std::slice; use std::sync::Arc; use std::time::Duration; @@ -11,19 +11,20 @@ use time::OffsetDateTime; use tokio::sync::{watch, RwLock}; use crate::options::SchedulerConfig; -use crate::update_file_store::UpdateFileStore; +use crate::snapshot::SnapshotJob; -use super::batch::Batch; +use super::batch::{Batch, BatchContent}; use super::error::Result; -use super::task::{Job, Task, TaskContent, TaskEvent, TaskId}; +use super::task::{Task, TaskContent, TaskEvent, TaskId}; use super::update_loop::UpdateLoop; -use super::{TaskFilter, TaskPerformer, TaskStore}; +use super::{BatchHandler, TaskFilter, TaskStore}; #[derive(Eq, Debug, Clone, Copy)] enum TaskType { DocumentAddition { number: usize }, DocumentUpdate { number: usize }, - Other, + IndexUpdate, + Dump, } /// Two tasks are equal if they have the same type. @@ -63,7 +64,7 @@ impl Ord for PendingTask { #[derive(Debug)] struct TaskList { - index: String, + id: TaskListIdentifier, tasks: BinaryHeap, } @@ -82,9 +83,9 @@ impl DerefMut for TaskList { } impl TaskList { - fn new(index: String) -> Self { + fn new(id: TaskListIdentifier) -> Self { Self { - index, + id, tasks: Default::default(), } } @@ -92,7 +93,7 @@ impl TaskList { impl PartialEq for TaskList { fn eq(&self, other: &Self) -> bool { - self.index == other.index + self.id == other.id } } @@ -100,11 +101,20 @@ impl Eq for TaskList {} impl Ord for TaskList { fn cmp(&self, other: &Self) -> Ordering { - match (self.peek(), other.peek()) { - (None, None) => Ordering::Equal, - (None, Some(_)) => Ordering::Less, - (Some(_), None) => Ordering::Greater, - (Some(lhs), Some(rhs)) => lhs.cmp(rhs), + match (&self.id, &other.id) { + (TaskListIdentifier::Index(_), TaskListIdentifier::Index(_)) => { + match (self.peek(), other.peek()) { + (None, None) => Ordering::Equal, + (None, Some(_)) => Ordering::Less, + (Some(_), None) => Ordering::Greater, + (Some(lhs), Some(rhs)) => lhs.cmp(rhs), + } + } + (TaskListIdentifier::Index(_), TaskListIdentifier::Dump) => Ordering::Less, + (TaskListIdentifier::Dump, TaskListIdentifier::Index(_)) => Ordering::Greater, + (TaskListIdentifier::Dump, TaskListIdentifier::Dump) => { + unreachable!("There should be only one Dump task list") + } } } } @@ -115,18 +125,41 @@ impl PartialOrd for TaskList { } } +#[derive(PartialEq, Eq, Hash, Debug, Clone)] +enum TaskListIdentifier { + Index(String), + Dump, +} + +impl From<&Task> for TaskListIdentifier { + fn from(task: &Task) -> Self { + match &task.content { + TaskContent::DocumentAddition { index_uid, .. } + | TaskContent::DocumentDeletion { index_uid, .. } + | TaskContent::SettingsUpdate { index_uid, .. } + | TaskContent::IndexDeletion { index_uid } + | TaskContent::IndexCreation { index_uid, .. } + | TaskContent::IndexUpdate { index_uid, .. } => { + TaskListIdentifier::Index(index_uid.as_str().to_string()) + } + TaskContent::Dump { .. } => TaskListIdentifier::Dump, + } + } +} + #[derive(Default)] struct TaskQueue { /// Maps index uids to their TaskList, for quick access - index_tasks: HashMap>>, + index_tasks: HashMap>>, /// A queue that orders TaskList by the priority of their fist update queue: BinaryHeap>>, } impl TaskQueue { fn insert(&mut self, task: Task) { - let uid = task.index_uid.into_inner(); let id = task.id; + let uid = TaskListIdentifier::from(&task); + let kind = match task.content { TaskContent::DocumentAddition { documents_count, @@ -142,7 +175,13 @@ impl TaskQueue { } => TaskType::DocumentUpdate { number: documents_count, }, - _ => TaskType::Other, + TaskContent::Dump { .. } => TaskType::Dump, + TaskContent::DocumentDeletion { .. } + | TaskContent::SettingsUpdate { .. } + | TaskContent::IndexDeletion { .. } + | TaskContent::IndexCreation { .. } + | TaskContent::IndexUpdate { .. } => TaskType::IndexUpdate, + _ => unreachable!("unhandled task type"), }; let task = PendingTask { kind, id }; @@ -160,7 +199,7 @@ impl TaskQueue { list.push(task); } Entry::Vacant(entry) => { - let mut task_list = TaskList::new(entry.key().to_owned()); + let mut task_list = TaskList::new(entry.key().clone()); task_list.push(task); let task_list = Arc::new(AtomicRefCell::new(task_list)); entry.insert(task_list.clone()); @@ -181,7 +220,7 @@ impl TaskQueue { // After being mutated, the head is reinserted to the correct position. self.queue.push(head); } else { - self.index_tasks.remove(&head.borrow().index); + self.index_tasks.remove(&head.borrow().id); } Some(result) @@ -193,11 +232,12 @@ impl TaskQueue { } pub struct Scheduler { - jobs: VecDeque, + // TODO: currently snapshots are non persistent tasks, and are treated differently. + snapshots: VecDeque, tasks: TaskQueue, store: TaskStore, - processing: Vec, + processing: Processing, next_fetched_task_id: TaskId, config: SchedulerConfig, /// Notifies the update loop that a new task was received @@ -205,14 +245,11 @@ pub struct Scheduler { } impl Scheduler { - pub fn new

( + pub fn new( store: TaskStore, - performer: Arc

, + performers: Vec>, mut config: SchedulerConfig, - ) -> Result>> - where - P: TaskPerformer, - { + ) -> Result>> { let (notifier, rcv) = watch::channel(()); let debounce_time = config.debounce_duration_sec; @@ -223,11 +260,11 @@ impl Scheduler { } let this = Self { - jobs: VecDeque::new(), + snapshots: VecDeque::new(), tasks: TaskQueue::default(), store, - processing: Vec::new(), + processing: Processing::Nothing, next_fetched_task_id: 0, config, notifier, @@ -240,7 +277,7 @@ impl Scheduler { let update_loop = UpdateLoop::new( this.clone(), - performer, + performers, debounce_time.filter(|&v| v > 0).map(Duration::from_secs), rcv, ); @@ -250,10 +287,6 @@ impl Scheduler { Ok(this) } - pub async fn dump(&self, path: &Path, file_store: UpdateFileStore) -> Result<()> { - self.store.dump(path, file_store).await - } - fn register_task(&mut self, task: Task) { assert!(!task.is_finished()); self.tasks.insert(task); @@ -261,7 +294,7 @@ impl Scheduler { /// Clears the processing list, this method should be called when the processing of a batch is finished. pub fn finish(&mut self) { - self.processing.clear(); + self.processing = Processing::Nothing; } pub fn notify(&self) { @@ -269,13 +302,27 @@ impl Scheduler { } fn notify_if_not_empty(&self) { - if !self.jobs.is_empty() || !self.tasks.is_empty() { + if !self.snapshots.is_empty() || !self.tasks.is_empty() { self.notify(); } } - pub async fn update_tasks(&self, tasks: Vec) -> Result> { - self.store.update_tasks(tasks).await + pub async fn update_tasks(&self, content: BatchContent) -> Result { + match content { + BatchContent::DocumentsAdditionBatch(tasks) => { + let tasks = self.store.update_tasks(tasks).await?; + Ok(BatchContent::DocumentsAdditionBatch(tasks)) + } + BatchContent::IndexUpdate(t) => { + let mut tasks = self.store.update_tasks(vec![t]).await?; + Ok(BatchContent::IndexUpdate(tasks.remove(0))) + } + BatchContent::Dump(t) => { + let mut tasks = self.store.update_tasks(vec![t]).await?; + Ok(BatchContent::Dump(tasks.remove(0))) + } + other => Ok(other), + } } pub async fn get_task(&self, id: TaskId, filter: Option) -> Result { @@ -294,32 +341,24 @@ impl Scheduler { pub async fn get_processing_tasks(&self) -> Result> { let mut tasks = Vec::new(); - for id in self.processing.iter() { - let task = self.store.get_task(*id, None).await?; + for id in self.processing.ids() { + let task = self.store.get_task(id, None).await?; tasks.push(task); } Ok(tasks) } - pub async fn schedule_job(&mut self, job: Job) { - self.jobs.push_back(job); + pub fn schedule_snapshot(&mut self, job: SnapshotJob) { + self.snapshots.push_back(job); self.notify(); } async fn fetch_pending_tasks(&mut self) -> Result<()> { - // We must NEVER re-enqueue an already processed task! It's content uuid would point to an unexisting file. - // - // TODO(marin): This may create some latency when the first batch lazy loads the pending updates. - let mut filter = TaskFilter::default(); - filter.filter_fn(|task| !task.is_finished()); - self.store - .list_tasks(Some(self.next_fetched_task_id), Some(filter), None) + .fetch_unfinished_tasks(Some(self.next_fetched_task_id)) .await? .into_iter() - // The tasks arrive in reverse order, and we need to insert them in order. - .rev() .for_each(|t| { self.next_fetched_task_id = t.id + 1; self.register_task(t); @@ -329,136 +368,199 @@ impl Scheduler { } /// Prepare the next batch, and set `processing` to the ids in that batch. - pub async fn prepare(&mut self) -> Result { + pub async fn prepare(&mut self) -> Result { // If there is a job to process, do it first. - if let Some(job) = self.jobs.pop_front() { + if let Some(job) = self.snapshots.pop_front() { // There is more work to do, notify the update loop self.notify_if_not_empty(); - return Ok(Pending::Job(job)); + let batch = Batch::new(None, BatchContent::Snapshot(job)); + return Ok(batch); } + // Try to fill the queue with pending tasks. self.fetch_pending_tasks().await?; - make_batch(&mut self.tasks, &mut self.processing, &self.config); + self.processing = make_batch(&mut self.tasks, &self.config); log::debug!("prepared batch with {} tasks", self.processing.len()); - if !self.processing.is_empty() { - let ids = std::mem::take(&mut self.processing); + if !self.processing.is_nothing() { + let (processing, mut content) = self + .store + .get_processing_tasks(std::mem::take(&mut self.processing)) + .await?; - let (ids, mut tasks) = self.store.get_pending_tasks(ids).await?; - - // The batch id is the id of the first update it contains - let id = match tasks.first() { + // The batch id is the id of the first update it contains. At this point we must have a + // valid batch that contains at least 1 task. + let id = match content.first() { Some(Task { id, .. }) => *id, _ => panic!("invalid batch"), }; - tasks.iter_mut().for_each(|t| { - t.events.push(TaskEvent::Batched { - batch_id: id, - timestamp: OffsetDateTime::now_utc(), - }) + content.push_event(TaskEvent::Batched { + batch_id: id, + timestamp: OffsetDateTime::now_utc(), }); - self.processing = ids; + self.processing = processing; - let batch = Batch { - id, - created_at: OffsetDateTime::now_utc(), - tasks, - }; + let batch = Batch::new(Some(id), content); // There is more work to do, notify the update loop self.notify_if_not_empty(); - Ok(Pending::Batch(batch)) + Ok(batch) } else { - Ok(Pending::Nothing) + Ok(Batch::empty()) } } } -#[derive(Debug)] -pub enum Pending { - Batch(Batch), - Job(Job), +#[derive(Debug, PartialEq)] +pub enum Processing { + DocumentAdditions(Vec), + IndexUpdate(TaskId), + Dump(TaskId), + /// Variant used when there is nothing to process. Nothing, } -fn make_batch(tasks: &mut TaskQueue, processing: &mut Vec, config: &SchedulerConfig) { - processing.clear(); +impl Default for Processing { + fn default() -> Self { + Self::Nothing + } +} - let mut doc_count = 0; - tasks.head_mut(|list| match list.peek().copied() { - Some(PendingTask { - kind: TaskType::Other, - id, - }) => { - processing.push(id); - list.pop(); +enum ProcessingIter<'a> { + Many(slice::Iter<'a, TaskId>), + Single(Option), +} + +impl<'a> Iterator for ProcessingIter<'a> { + type Item = TaskId; + + fn next(&mut self) -> Option { + match self { + ProcessingIter::Many(iter) => iter.next().copied(), + ProcessingIter::Single(val) => val.take(), } - Some(PendingTask { kind, .. }) => loop { - match list.peek() { - Some(pending) if pending.kind == kind => { - // We always need to process at least one task for the scheduler to make progress. - if processing.len() >= config.max_batch_size.unwrap_or(usize::MAX).max(1) { - break; - } - let pending = list.pop().unwrap(); - processing.push(pending.id); + } +} - // We add the number of documents to the count if we are scheduling document additions and - // stop adding if we already have enough. - // - // We check that bound only after adding the current task to the batch, so that a batch contains at least one task. - match pending.kind { - TaskType::DocumentUpdate { number } - | TaskType::DocumentAddition { number } => { - doc_count += number; +impl Processing { + fn is_nothing(&self) -> bool { + matches!(self, Processing::Nothing) + } - if doc_count >= config.max_documents_per_batch.unwrap_or(usize::MAX) { + pub fn ids(&self) -> impl Iterator + '_ { + match self { + Processing::DocumentAdditions(v) => ProcessingIter::Many(v.iter()), + Processing::IndexUpdate(id) | Processing::Dump(id) => ProcessingIter::Single(Some(*id)), + Processing::Nothing => ProcessingIter::Single(None), + } + } + + pub fn len(&self) -> usize { + match self { + Processing::DocumentAdditions(v) => v.len(), + Processing::IndexUpdate(_) | Processing::Dump(_) => 1, + Processing::Nothing => 0, + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +fn make_batch(tasks: &mut TaskQueue, config: &SchedulerConfig) -> Processing { + let mut doc_count = 0; + tasks + .head_mut(|list| match list.peek().copied() { + Some(PendingTask { + kind: TaskType::IndexUpdate, + id, + }) => { + list.pop(); + Processing::IndexUpdate(id) + } + Some(PendingTask { + kind: TaskType::Dump, + id, + }) => { + list.pop(); + Processing::Dump(id) + } + Some(PendingTask { kind, .. }) => { + let mut task_list = Vec::new(); + loop { + match list.peek() { + Some(pending) if pending.kind == kind => { + // We always need to process at least one task for the scheduler to make progress. + if task_list.len() >= config.max_batch_size.unwrap_or(usize::MAX).max(1) + { break; } + let pending = list.pop().unwrap(); + task_list.push(pending.id); + + // We add the number of documents to the count if we are scheduling document additions and + // stop adding if we already have enough. + // + // We check that bound only after adding the current task to the batch, so that a batch contains at least one task. + match pending.kind { + TaskType::DocumentUpdate { number } + | TaskType::DocumentAddition { number } => { + doc_count += number; + + if doc_count + >= config.max_documents_per_batch.unwrap_or(usize::MAX) + { + break; + } + } + _ => (), + } } - _ => (), + _ => break, } } - _ => break, + Processing::DocumentAdditions(task_list) } - }, - None => (), - }); + None => Processing::Nothing, + }) + .unwrap_or(Processing::Nothing) } #[cfg(test)] mod test { + use meilisearch_types::index_uid::IndexUid; use milli::update::IndexDocumentsMethod; use uuid::Uuid; - use crate::{index_resolver::IndexUid, tasks::task::TaskContent}; + use crate::tasks::task::TaskContent; use super::*; - fn gen_task(id: TaskId, index_uid: &str, content: TaskContent) -> Task { + fn gen_task(id: TaskId, content: TaskContent) -> Task { Task { id, - index_uid: IndexUid::new_unchecked(index_uid), content, events: vec![], } } #[test] + #[rustfmt::skip] fn register_updates_multiples_indexes() { let mut queue = TaskQueue::default(); - queue.insert(gen_task(0, "test1", TaskContent::IndexDeletion)); - queue.insert(gen_task(1, "test2", TaskContent::IndexDeletion)); - queue.insert(gen_task(2, "test2", TaskContent::IndexDeletion)); - queue.insert(gen_task(3, "test2", TaskContent::IndexDeletion)); - queue.insert(gen_task(4, "test1", TaskContent::IndexDeletion)); - queue.insert(gen_task(5, "test1", TaskContent::IndexDeletion)); - queue.insert(gen_task(6, "test2", TaskContent::IndexDeletion)); + queue.insert(gen_task(0, TaskContent::IndexDeletion { index_uid: IndexUid::new_unchecked("test1") })); + queue.insert(gen_task(1, TaskContent::IndexDeletion { index_uid: IndexUid::new_unchecked("test2") })); + queue.insert(gen_task(2, TaskContent::IndexDeletion { index_uid: IndexUid::new_unchecked("test2") })); + queue.insert(gen_task(3, TaskContent::IndexDeletion { index_uid: IndexUid::new_unchecked("test2") })); + queue.insert(gen_task(4, TaskContent::IndexDeletion { index_uid: IndexUid::new_unchecked("test1") })); + queue.insert(gen_task(5, TaskContent::IndexDeletion { index_uid: IndexUid::new_unchecked("test1") })); + queue.insert(gen_task(6, TaskContent::IndexDeletion { index_uid: IndexUid::new_unchecked("test2") })); let test1_tasks = queue .head_mut(|tasks| tasks.drain().map(|t| t.id).collect::>()) @@ -476,50 +578,54 @@ mod test { assert!(queue.queue.is_empty()); } - #[test] - fn test_make_batch() { - let mut queue = TaskQueue::default(); - let content = TaskContent::DocumentAddition { + fn gen_doc_addition_task_content(index_uid: &str) -> TaskContent { + TaskContent::DocumentAddition { content_uuid: Uuid::new_v4(), merge_strategy: IndexDocumentsMethod::ReplaceDocuments, primary_key: Some("test".to_string()), documents_count: 0, allow_index_creation: true, - }; - queue.insert(gen_task(0, "test1", content.clone())); - queue.insert(gen_task(1, "test2", content.clone())); - queue.insert(gen_task(2, "test2", TaskContent::IndexDeletion)); - queue.insert(gen_task(3, "test2", content.clone())); - queue.insert(gen_task(4, "test1", content.clone())); - queue.insert(gen_task(5, "test1", TaskContent::IndexDeletion)); - queue.insert(gen_task(6, "test2", content.clone())); - queue.insert(gen_task(7, "test1", content)); + index_uid: IndexUid::new_unchecked(index_uid), + } + } - let mut batch = Vec::new(); + #[test] + #[rustfmt::skip] + fn test_make_batch() { + let mut queue = TaskQueue::default(); + queue.insert(gen_task(0, gen_doc_addition_task_content("test1"))); + queue.insert(gen_task(1, gen_doc_addition_task_content("test2"))); + queue.insert(gen_task(2, TaskContent::IndexDeletion { index_uid: IndexUid::new_unchecked("test2")})); + queue.insert(gen_task(3, gen_doc_addition_task_content("test2"))); + queue.insert(gen_task(4, gen_doc_addition_task_content("test1"))); + queue.insert(gen_task(5, TaskContent::IndexDeletion { index_uid: IndexUid::new_unchecked("test1")})); + queue.insert(gen_task(6, gen_doc_addition_task_content("test2"))); + queue.insert(gen_task(7, gen_doc_addition_task_content("test1"))); + queue.insert(gen_task(8, TaskContent::Dump { uid: "adump".to_owned() })); let config = SchedulerConfig::default(); - make_batch(&mut queue, &mut batch, &config); - assert_eq!(batch, &[0, 4]); - batch.clear(); - make_batch(&mut queue, &mut batch, &config); - assert_eq!(batch, &[1]); + // Make sure that the dump is processed before everybody else. + let batch = make_batch(&mut queue, &config); + assert_eq!(batch, Processing::Dump(8)); - batch.clear(); - make_batch(&mut queue, &mut batch, &config); - assert_eq!(batch, &[2]); + let batch = make_batch(&mut queue, &config); + assert_eq!(batch, Processing::DocumentAdditions(vec![0, 4])); - batch.clear(); - make_batch(&mut queue, &mut batch, &config); - assert_eq!(batch, &[3, 6]); + let batch = make_batch(&mut queue, &config); + assert_eq!(batch, Processing::DocumentAdditions(vec![1])); - batch.clear(); - make_batch(&mut queue, &mut batch, &config); - assert_eq!(batch, &[5]); + let batch = make_batch(&mut queue, &config); + assert_eq!(batch, Processing::IndexUpdate(2)); - batch.clear(); - make_batch(&mut queue, &mut batch, &config); - assert_eq!(batch, &[7]); + let batch = make_batch(&mut queue, &config); + assert_eq!(batch, Processing::DocumentAdditions(vec![3, 6])); + + let batch = make_batch(&mut queue, &config); + assert_eq!(batch, Processing::IndexUpdate(5)); + + let batch = make_batch(&mut queue, &config); + assert_eq!(batch, Processing::DocumentAdditions(vec![7])); assert!(queue.is_empty()); } diff --git a/meilisearch-lib/src/tasks/task.rs b/meilisearch-lib/src/tasks/task.rs index ecbd4ca62..bd5579151 100644 --- a/meilisearch-lib/src/tasks/task.rs +++ b/meilisearch-lib/src/tasks/task.rs @@ -1,20 +1,14 @@ -use std::path::PathBuf; - -use meilisearch_error::ResponseError; +use meilisearch_types::error::ResponseError; +use meilisearch_types::index_uid::IndexUid; use milli::update::{DocumentAdditionResult, IndexDocumentsMethod}; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; -use tokio::sync::oneshot; use uuid::Uuid; use super::batch::BatchId; -use crate::{ - index::{Settings, Unchecked}, - index_resolver::{error::IndexResolverError, IndexUid}, - snapshot::SnapshotJob, -}; +use crate::index::{Settings, Unchecked}; -pub type TaskId = u64; +pub type TaskId = u32; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(test, derive(proptest_derive::Arbitrary))] @@ -52,7 +46,7 @@ pub enum TaskEvent { #[serde(with = "time::serde::rfc3339")] OffsetDateTime, ), - Succeded { + Succeeded { result: TaskResult, #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] #[serde(with = "time::serde::rfc3339")] @@ -66,6 +60,22 @@ pub enum TaskEvent { }, } +impl TaskEvent { + pub fn succeeded(result: TaskResult) -> Self { + Self::Succeeded { + result, + timestamp: OffsetDateTime::now_utc(), + } + } + + pub fn failed(error: impl Into) -> Self { + Self::Failed { + error: error.into(), + timestamp: OffsetDateTime::now_utc(), + } + } +} + /// A task represents an operation that Meilisearch must do. /// It's stored on disk and executed from the lowest to highest Task id. /// Everytime a new task is created it has a higher Task id than the previous one. @@ -74,7 +84,10 @@ pub enum TaskEvent { #[cfg_attr(test, derive(proptest_derive::Arbitrary))] pub struct Task { pub id: TaskId, - pub index_uid: IndexUid, + /// The name of the index the task is targeting. If it isn't targeting any index (i.e Dump task) + /// then this is None + // TODO: when next forward breaking dumps, it would be a good idea to move this field inside of + // the TaskContent. pub content: TaskContent, pub events: Vec, } @@ -84,7 +97,10 @@ impl Task { /// A task is finished when its last state is either `Succeeded` or `Failed`. pub fn is_finished(&self) -> bool { self.events.last().map_or(false, |event| { - matches!(event, TaskEvent::Succeded { .. } | TaskEvent::Failed { .. }) + matches!( + event, + TaskEvent::Succeeded { .. } | TaskEvent::Failed { .. } + ) }) } @@ -98,32 +114,17 @@ impl Task { _ => None, } } -} -/// A job is like a volatile priority `Task`. -/// It should be processed as fast as possible and is not stored on disk. -/// This means, when Meilisearch is closed all your unprocessed jobs will disappear. -#[derive(Debug, derivative::Derivative)] -#[derivative(PartialEq)] -pub enum Job { - Dump { - #[derivative(PartialEq = "ignore")] - ret: oneshot::Sender, IndexResolverError>>, - path: PathBuf, - }, - Snapshot(#[derivative(PartialEq = "ignore")] SnapshotJob), - Empty, -} - -impl Default for Job { - fn default() -> Self { - Self::Empty - } -} - -impl Job { - pub fn take(&mut self) -> Self { - std::mem::take(self) + pub fn index_uid(&self) -> Option<&str> { + match &self.content { + TaskContent::DocumentAddition { index_uid, .. } + | TaskContent::DocumentDeletion { index_uid, .. } + | TaskContent::SettingsUpdate { index_uid, .. } + | TaskContent::IndexDeletion { index_uid } + | TaskContent::IndexCreation { index_uid, .. } + | TaskContent::IndexUpdate { index_uid, .. } => Some(index_uid.as_str()), + TaskContent::Dump { .. } => None, + } } } @@ -139,6 +140,7 @@ pub enum DocumentDeletion { #[allow(clippy::large_enum_variant)] pub enum TaskContent { DocumentAddition { + index_uid: IndexUid, #[cfg_attr(test, proptest(value = "Uuid::new_v4()"))] content_uuid: Uuid, #[cfg_attr(test, proptest(strategy = "test::index_document_method_strategy()"))] @@ -147,20 +149,31 @@ pub enum TaskContent { documents_count: usize, allow_index_creation: bool, }, - DocumentDeletion(DocumentDeletion), + DocumentDeletion { + index_uid: IndexUid, + deletion: DocumentDeletion, + }, SettingsUpdate { + index_uid: IndexUid, settings: Settings, /// Indicates whether the task was a deletion is_deletion: bool, allow_index_creation: bool, }, - IndexDeletion, + IndexDeletion { + index_uid: IndexUid, + }, IndexCreation { + index_uid: IndexUid, primary_key: Option, }, IndexUpdate { + index_uid: IndexUid, primary_key: Option, }, + Dump { + uid: String, + }, } #[cfg(test)] diff --git a/meilisearch-lib/src/tasks/task_store/mod.rs b/meilisearch-lib/src/tasks/task_store/mod.rs index bdcd13f37..e2b01afb8 100644 --- a/meilisearch-lib/src/tasks/task_store/mod.rs +++ b/meilisearch-lib/src/tasks/task_store/mod.rs @@ -9,10 +9,11 @@ use log::debug; use milli::heed::{Env, RwTxn}; use time::OffsetDateTime; +use super::batch::BatchContent; use super::error::TaskError; +use super::scheduler::Processing; use super::task::{Task, TaskContent, TaskId}; use super::Result; -use crate::index_resolver::IndexUid; use crate::tasks::task::TaskEvent; use crate::update_file_store::UpdateFileStore; @@ -30,10 +31,17 @@ pub struct TaskFilter { impl TaskFilter { fn pass(&self, task: &Task) -> bool { - self.indexes - .as_ref() - .map(|indexes| indexes.contains(&*task.index_uid)) - .unwrap_or(true) + match task.index_uid() { + Some(index_uid) => self + .indexes + .as_ref() + .map_or(true, |indexes| indexes.contains(index_uid)), + None => false, + } + } + + fn filtered_indexes(&self) -> Option<&HashSet> { + self.indexes.as_ref() } /// Adds an index to the filter, so the filter must match this index. @@ -66,7 +74,7 @@ impl TaskStore { Ok(Self { store }) } - pub async fn register(&self, index_uid: IndexUid, content: TaskContent) -> Result { + pub async fn register(&self, content: TaskContent) -> Result { debug!("registering update: {:?}", content); let store = self.store.clone(); let task = tokio::task::spawn_blocking(move || -> Result { @@ -75,7 +83,6 @@ impl TaskStore { let created_at = TaskEvent::Created(OffsetDateTime::now_utc()); let task = Task { id: next_task_id, - index_uid, content, events: vec![created_at], }; @@ -114,19 +121,44 @@ impl TaskStore { } } - pub async fn get_pending_tasks(&self, ids: Vec) -> Result<(Vec, Vec)> { + /// This methods takes a `Processing` which contains the next task ids to process, and returns + /// the coresponding tasks along with the ownership to the passed processing. + /// + /// We need get_processing_tasks to take ownership over `Processing` because we need it to be + /// valid for 'static. + pub async fn get_processing_tasks( + &self, + processing: Processing, + ) -> Result<(Processing, BatchContent)> { let store = self.store.clone(); let tasks = tokio::task::spawn_blocking(move || -> Result<_> { - let mut tasks = Vec::new(); let txn = store.rtxn()?; - for id in ids.iter() { - let task = store - .get(&txn, *id)? - .ok_or(TaskError::UnexistingTask(*id))?; - tasks.push(task); - } - Ok((ids, tasks)) + let content = match processing { + Processing::DocumentAdditions(ref ids) => { + let mut tasks = Vec::new(); + + for id in ids.iter() { + let task = store + .get(&txn, *id)? + .ok_or(TaskError::UnexistingTask(*id))?; + tasks.push(task); + } + BatchContent::DocumentsAdditionBatch(tasks) + } + Processing::IndexUpdate(id) => { + let task = store.get(&txn, id)?.ok_or(TaskError::UnexistingTask(id))?; + BatchContent::IndexUpdate(task) + } + Processing::Dump(id) => { + let task = store.get(&txn, id)?.ok_or(TaskError::UnexistingTask(id))?; + debug_assert!(matches!(task.content, TaskContent::Dump { .. })); + BatchContent::Dump(task) + } + Processing::Nothing => BatchContent::Empty, + }; + + Ok((processing, content)) }) .await??; @@ -152,6 +184,17 @@ impl TaskStore { Ok(tasks) } + pub async fn fetch_unfinished_tasks(&self, offset: Option) -> Result> { + let store = self.store.clone(); + + tokio::task::spawn_blocking(move || { + let txn = store.rtxn()?; + let tasks = store.fetch_unfinished_tasks(&txn, offset)?; + Ok(tasks) + }) + .await? + } + pub async fn list_tasks( &self, offset: Option, @@ -169,13 +212,14 @@ impl TaskStore { } pub async fn dump( - &self, + env: Arc, dir_path: impl AsRef, update_file_store: UpdateFileStore, ) -> Result<()> { + let store = Self::new(env)?; let update_dir = dir_path.as_ref().join("updates"); let updates_file = update_dir.join("data.jsonl"); - let tasks = self.list_tasks(None, None, None).await?; + let tasks = store.list_tasks(None, None, None).await?; let dir_path = dir_path.as_ref().to_path_buf(); tokio::task::spawn_blocking(move || -> Result<()> { @@ -223,10 +267,11 @@ impl TaskStore { #[cfg(test)] pub mod test { - use crate::tasks::task_store::store::test::tmp_env; + use crate::tasks::{scheduler::Processing, task_store::store::test::tmp_env}; use super::*; + use meilisearch_types::index_uid::IndexUid; use nelson::Mocker; use proptest::{ strategy::Strategy, @@ -252,6 +297,14 @@ pub mod test { Ok(Self::Real(TaskStore::new(env)?)) } + pub async fn dump( + env: Arc, + path: impl AsRef, + update_file_store: UpdateFileStore, + ) -> Result<()> { + TaskStore::dump(env, path, update_file_store).await + } + pub fn mock(mocker: Mocker) -> Self { Self::Mock(Arc::new(mocker)) } @@ -272,16 +325,23 @@ pub mod test { } } - pub async fn get_pending_tasks( + pub async fn get_processing_tasks( &self, - tasks: Vec, - ) -> Result<(Vec, Vec)> { + tasks: Processing, + ) -> Result<(Processing, BatchContent)> { match self { - Self::Real(s) => s.get_pending_tasks(tasks).await, + Self::Real(s) => s.get_processing_tasks(tasks).await, Self::Mock(m) => unsafe { m.get("get_pending_task").call(tasks) }, } } + pub async fn fetch_unfinished_tasks(&self, from: Option) -> Result> { + match self { + Self::Real(s) => s.fetch_unfinished_tasks(from).await, + Self::Mock(m) => unsafe { m.get("fetch_unfinished_tasks").call(from) }, + } + } + pub async fn list_tasks( &self, from: Option, @@ -294,20 +354,9 @@ pub mod test { } } - pub async fn dump( - &self, - path: impl AsRef, - update_file_store: UpdateFileStore, - ) -> Result<()> { + pub async fn register(&self, content: TaskContent) -> Result { match self { - Self::Real(s) => s.dump(path, update_file_store).await, - Self::Mock(m) => unsafe { m.get("dump").call((path, update_file_store)) }, - } - } - - pub async fn register(&self, index_uid: IndexUid, content: TaskContent) -> Result { - match self { - Self::Real(s) => s.register(index_uid, content).await, + Self::Real(s) => s.register(content).await, Self::Mock(_m) => todo!(), } } @@ -335,14 +384,16 @@ pub mod test { let gen_task = |id: TaskId| Task { id, - index_uid: IndexUid::new_unchecked("test"), - content: TaskContent::IndexCreation { primary_key: None }, + content: TaskContent::IndexCreation { + primary_key: None, + index_uid: IndexUid::new_unchecked("test"), + }, events: Vec::new(), }; let mut runner = TestRunner::new(Config::default()); runner - .run(&(0..100u64).prop_map(gen_task), |task| { + .run(&(0..100u32).prop_map(gen_task), |task| { let mut txn = store.wtxn().unwrap(); let previous_id = store.next_task_id(&mut txn).unwrap(); diff --git a/meilisearch-lib/src/tasks/task_store/store.rs b/meilisearch-lib/src/tasks/task_store/store.rs index 4ff986d8b..9dfe61c55 100644 --- a/meilisearch-lib/src/tasks/task_store/store.rs +++ b/meilisearch-lib/src/tasks/task_store/store.rs @@ -1,62 +1,30 @@ #[allow(clippy::upper_case_acronyms)] -type BEU64 = milli::heed::zerocopy::U64; -const UID_TASK_IDS: &str = "uid_task_id"; +type BEU32 = milli::heed::zerocopy::U32; + +const INDEX_UIDS_TASK_IDS: &str = "index-uids-task-ids"; const TASKS: &str = "tasks"; -use std::borrow::Cow; -use std::collections::BinaryHeap; -use std::convert::TryInto; -use std::mem::size_of; -use std::ops::Range; +use std::collections::HashSet; +use std::ops::Bound::{Excluded, Unbounded}; use std::result::Result as StdResult; use std::sync::Arc; -use milli::heed::types::{ByteSlice, OwnedType, SerdeJson, Unit}; -use milli::heed::{BytesDecode, BytesEncode, Database, Env, RoTxn, RwTxn}; +use milli::heed::types::{OwnedType, SerdeJson, Str}; +use milli::heed::{Database, Env, RoTxn, RwTxn}; +use milli::heed_codec::RoaringBitmapCodec; +use roaring::RoaringBitmap; use crate::tasks::task::{Task, TaskId}; use super::super::Result; - use super::TaskFilter; -enum IndexUidTaskIdCodec {} - -impl<'a> BytesEncode<'a> for IndexUidTaskIdCodec { - type EItem = (&'a str, TaskId); - - fn bytes_encode((s, id): &'a Self::EItem) -> Option> { - let size = s.len() + std::mem::size_of::() + 1; - if size > 512 { - return None; - } - let mut b = Vec::with_capacity(size); - b.extend_from_slice(s.as_bytes()); - // null terminate the string - b.push(0); - b.extend_from_slice(&id.to_be_bytes()); - Some(Cow::Owned(b)) - } -} - -impl<'a> BytesDecode<'a> for IndexUidTaskIdCodec { - type DItem = (&'a str, TaskId); - - fn bytes_decode(bytes: &'a [u8]) -> Option { - let len = bytes.len(); - let s_end = len.checked_sub(size_of::())?.checked_sub(1)?; - let str_bytes = &bytes[..s_end]; - let str = std::str::from_utf8(str_bytes).ok()?; - let id = TaskId::from_be_bytes(bytes[(len - size_of::())..].try_into().ok()?); - Some((str, id)) - } -} - pub struct Store { env: Arc, - uids_task_ids: Database, - tasks: Database, SerdeJson>, + /// Maps an index uid to the set of tasks ids associated to it. + index_uid_task_ids: Database, + tasks: Database, SerdeJson>, } impl Drop for Store { @@ -74,12 +42,12 @@ impl Store { /// You want to patch all un-finished tasks and put them in your pending /// queue with the `reset_and_return_unfinished_update` method. pub fn new(env: Arc) -> Result { - let uids_task_ids = env.create_database(Some(UID_TASK_IDS))?; + let index_uid_task_ids = env.create_database(Some(INDEX_UIDS_TASK_IDS))?; let tasks = env.create_database(Some(TASKS))?; Ok(Self { env, - uids_task_ids, + index_uid_task_ids, tasks, }) } @@ -107,132 +75,115 @@ impl Store { } pub fn put(&self, txn: &mut RwTxn, task: &Task) -> Result<()> { - self.tasks.put(txn, &BEU64::new(task.id), task)?; - self.uids_task_ids - .put(txn, &(&task.index_uid, task.id), &())?; + self.tasks.put(txn, &BEU32::new(task.id), task)?; + // only add the task to the indexes index if it has an index_uid + if let Some(index_uid) = task.index_uid() { + let mut tasks_set = self + .index_uid_task_ids + .get(txn, index_uid)? + .unwrap_or_default(); + + tasks_set.insert(task.id); + + self.index_uid_task_ids.put(txn, index_uid, &tasks_set)?; + } Ok(()) } pub fn get(&self, txn: &RoTxn, id: TaskId) -> Result> { - let task = self.tasks.get(txn, &BEU64::new(id))?; + let task = self.tasks.get(txn, &BEU32::new(id))?; Ok(task) } - pub fn list_tasks<'a>( + /// Returns the unfinished tasks starting from the given taskId in ascending order. + pub fn fetch_unfinished_tasks(&self, txn: &RoTxn, from: Option) -> Result> { + // We must NEVER re-enqueue an already processed task! It's content uuid would point to an unexisting file. + // + // TODO(marin): This may create some latency when the first batch lazy loads the pending updates. + let from = from.unwrap_or_default(); + + let result: StdResult, milli::heed::Error> = self + .tasks + .range(txn, &(BEU32::new(from)..))? + .map(|r| r.map(|(_, t)| t)) + .filter(|result| result.as_ref().map_or(true, |t| !t.is_finished())) + .collect(); + + result.map_err(Into::into) + } + + /// Returns all the tasks starting from the given taskId and going in descending order. + pub fn list_tasks( &self, - txn: &'a RoTxn, + txn: &RoTxn, from: Option, filter: Option, limit: Option, ) -> Result> { - let from = from.unwrap_or_default(); - let range = from..limit - .map(|limit| (limit as u64).saturating_add(from)) - .unwrap_or(u64::MAX); - let iter: Box>> = match filter { - Some( - ref filter @ TaskFilter { - indexes: Some(_), .. - }, - ) => { - let iter = self - .compute_candidates(txn, filter, range)? - .into_iter() - .filter_map(|id| self.tasks.get(txn, &BEU64::new(id)).transpose()); - - Box::new(iter) - } - _ => Box::new( - self.tasks - .rev_range(txn, &(BEU64::new(range.start)..BEU64::new(range.end)))? - .map(|r| r.map(|(_, t)| t)), - ), + let from = match from { + Some(from) => from, + None => self.tasks.last(txn)?.map_or(0, |(id, _)| id.get()), }; - let apply_fitler = |task: &StdResult<_, milli::heed::Error>| match task { - Ok(ref t) => filter + let filter_fn = |task: &Task| { + filter .as_ref() - .and_then(|filter| filter.filter_fn.as_ref()) - .map(|f| f(t)) - .unwrap_or(true), - Err(_) => true, + .and_then(|f| f.filter_fn.as_ref()) + .map_or(true, |f| f(task)) }; - // Collect 'limit' task if it exists or all of them. - let tasks = iter - .filter(apply_fitler) - .take(limit.unwrap_or(usize::MAX)) - .try_fold::<_, _, StdResult<_, milli::heed::Error>>(Vec::new(), |mut v, task| { - v.push(task?); - Ok(v) - })?; - Ok(tasks) + let result: Result> = match filter.as_ref().and_then(|f| f.filtered_indexes()) { + Some(indexes) => self + .compute_candidates(txn, indexes, from)? + .filter(|result| result.as_ref().map_or(true, filter_fn)) + .take(limit.unwrap_or(usize::MAX)) + .collect(), + None => self + .tasks + .rev_range(txn, &(..=BEU32::new(from)))? + .map(|r| r.map(|(_, t)| t).map_err(Into::into)) + .filter(|result| result.as_ref().map_or(true, filter_fn)) + .take(limit.unwrap_or(usize::MAX)) + .collect(), + }; + + result.map_err(Into::into) } - fn compute_candidates( - &self, - txn: &milli::heed::RoTxn, - filter: &TaskFilter, - range: Range, - ) -> Result> { - let mut candidates = BinaryHeap::new(); - if let Some(ref indexes) = filter.indexes { - for index in indexes { - // We need to prefix search the null terminated string to make sure that we only - // get exact matches for the index, and not other uids that would share the same - // prefix, i.e test and test1. - let mut index_uid = index.as_bytes().to_vec(); - index_uid.push(0); + fn compute_candidates<'a>( + &'a self, + txn: &'a RoTxn, + indexes: &HashSet, + from: TaskId, + ) -> Result> + 'a> { + let mut candidates = RoaringBitmap::new(); - self.uids_task_ids - .remap_key_type::() - .rev_prefix_iter(txn, &index_uid)? - .map(|entry| -> StdResult<_, milli::heed::Error> { - let (key, _) = entry?; - let (_, id) = IndexUidTaskIdCodec::bytes_decode(key) - .ok_or(milli::heed::Error::Decoding)?; - Ok(id) - }) - .skip_while(|entry| { - entry - .as_ref() - .ok() - // we skip all elements till we enter in the range - .map(|key| !range.contains(key)) - // if we encounter an error we returns true to collect it later - .unwrap_or(true) - }) - .take_while(|entry| { - entry - .as_ref() - .ok() - // as soon as we are out of the range we exit - .map(|key| range.contains(key)) - // if we encounter an error we returns true to collect it later - .unwrap_or(true) - }) - .try_for_each::<_, StdResult<(), milli::heed::Error>>(|id| { - candidates.push(id?); - Ok(()) - })?; + for index_uid in indexes { + if let Some(tasks_set) = self.index_uid_task_ids.get(txn, index_uid)? { + candidates |= tasks_set; } } - Ok(candidates) + candidates.remove_range((Excluded(from), Unbounded)); + + let iter = candidates + .into_iter() + .rev() + .filter_map(|id| self.get(txn, id).transpose()); + + Ok(iter) } } #[cfg(test)] pub mod test { use itertools::Itertools; + use meilisearch_types::index_uid::IndexUid; use milli::heed::EnvOpenOptions; use nelson::Mocker; - use proptest::collection::vec; - use proptest::prelude::*; use tempfile::TempDir; - use crate::index_resolver::IndexUid; use crate::tasks::task::TaskContent; use super::*; @@ -303,9 +254,20 @@ pub mod test { } } - pub fn list_tasks<'a>( + pub fn fetch_unfinished_tasks( &self, - txn: &'a RoTxn, + txn: &RoTxn, + from: Option, + ) -> Result> { + match self { + MockStore::Real(index) => index.fetch_unfinished_tasks(txn, from), + MockStore::Fake(_) => todo!(), + } + } + + pub fn list_tasks( + &self, + txn: &RoTxn, from: Option, filter: Option, limit: Option, @@ -325,8 +287,9 @@ pub mod test { let tasks = (0..100) .map(|_| Task { id: rand::random(), - index_uid: IndexUid::new_unchecked("test"), - content: TaskContent::IndexDeletion, + content: TaskContent::IndexDeletion { + index_uid: IndexUid::new_unchecked("test"), + }, events: vec![], }) .collect::>(); @@ -356,15 +319,17 @@ pub mod test { let task_1 = Task { id: 1, - index_uid: IndexUid::new_unchecked("test"), - content: TaskContent::IndexDeletion, + content: TaskContent::IndexDeletion { + index_uid: IndexUid::new_unchecked("test"), + }, events: vec![], }; let task_2 = Task { id: 0, - index_uid: IndexUid::new_unchecked("test1"), - content: TaskContent::IndexDeletion, + content: TaskContent::IndexDeletion { + index_uid: IndexUid::new_unchecked("test1"), + }, events: vec![], }; @@ -379,19 +344,21 @@ pub mod test { txn.abort().unwrap(); assert_eq!(tasks.len(), 1); - assert_eq!(&*tasks.first().unwrap().index_uid, "test"); + assert_eq!(tasks.first().as_ref().unwrap().index_uid().unwrap(), "test"); // same thing but invert the ids let task_1 = Task { id: 0, - index_uid: IndexUid::new_unchecked("test"), - content: TaskContent::IndexDeletion, + content: TaskContent::IndexDeletion { + index_uid: IndexUid::new_unchecked("test"), + }, events: vec![], }; let task_2 = Task { id: 1, - index_uid: IndexUid::new_unchecked("test1"), - content: TaskContent::IndexDeletion, + content: TaskContent::IndexDeletion { + index_uid: IndexUid::new_unchecked("test1"), + }, events: vec![], }; @@ -405,28 +372,9 @@ pub mod test { let tasks = store.list_tasks(&txn, None, Some(filter), None).unwrap(); assert_eq!(tasks.len(), 1); - assert_eq!(&*tasks.first().unwrap().index_uid, "test"); - } - - proptest! { - #[test] - fn encode_decode_roundtrip(index_uid in any::(), task_id in 0..TaskId::MAX) { - let value = (index_uid.as_ref(), task_id); - let bytes = IndexUidTaskIdCodec::bytes_encode(&value).unwrap(); - let (index, id) = IndexUidTaskIdCodec::bytes_decode(bytes.as_ref()).unwrap(); - assert_eq!(&*index_uid, index); - assert_eq!(task_id, id); - } - - #[test] - fn encode_doesnt_crash(index_uid in "\\PC*", task_id in 0..TaskId::MAX) { - let value = (index_uid.as_ref(), task_id); - IndexUidTaskIdCodec::bytes_encode(&value); - } - - #[test] - fn decode_doesnt_crash(bytes in vec(any::(), 0..1000)) { - IndexUidTaskIdCodec::bytes_decode(&bytes); - } + assert_eq!( + &*tasks.first().as_ref().unwrap().index_uid().unwrap(), + "test" + ); } } diff --git a/meilisearch-lib/src/tasks/update_loop.rs b/meilisearch-lib/src/tasks/update_loop.rs index b09811721..01e88755a 100644 --- a/meilisearch-lib/src/tasks/update_loop.rs +++ b/meilisearch-lib/src/tasks/update_loop.rs @@ -7,33 +7,29 @@ use tokio::time::interval_at; use super::batch::Batch; use super::error::Result; -use super::scheduler::Pending; -use super::{Scheduler, TaskPerformer}; +use super::{BatchHandler, Scheduler}; use crate::tasks::task::TaskEvent; /// The update loop sequentially performs batches of updates by asking the scheduler for a batch, /// and handing it to the `TaskPerformer`. -pub struct UpdateLoop { +pub struct UpdateLoop { scheduler: Arc>, - performer: Arc

, + performers: Vec>, notifier: Option>, debounce_duration: Option, } -impl

UpdateLoop

-where - P: TaskPerformer + Send + Sync + 'static, -{ +impl UpdateLoop { pub fn new( scheduler: Arc>, - performer: Arc

, + performers: Vec>, debuf_duration: Option, notifier: watch::Receiver<()>, ) -> Self { Self { scheduler, - performer, + performers, debounce_duration: debuf_duration, notifier: Some(notifier), } @@ -59,34 +55,29 @@ where } async fn process_next_batch(&self) -> Result<()> { - let pending = { self.scheduler.write().await.prepare().await? }; - match pending { - Pending::Batch(mut batch) => { - for task in &mut batch.tasks { - task.events - .push(TaskEvent::Processing(OffsetDateTime::now_utc())); - } + let mut batch = { self.scheduler.write().await.prepare().await? }; + let performer = self + .performers + .iter() + .find(|p| p.accept(&batch)) + .expect("No performer found for batch") + .clone(); - batch.tasks = { - self.scheduler - .read() - .await - .update_tasks(batch.tasks) - .await? - }; + batch + .content + .push_event(TaskEvent::Processing(OffsetDateTime::now_utc())); - let performer = self.performer.clone(); + batch.content = { + self.scheduler + .read() + .await + .update_tasks(batch.content) + .await? + }; - let batch = performer.process_batch(batch).await; + let batch = performer.process_batch(batch).await; - self.handle_batch_result(batch).await?; - } - Pending::Job(job) => { - let performer = self.performer.clone(); - performer.process_job(job).await; - } - Pending::Nothing => (), - } + self.handle_batch_result(batch, performer).await?; Ok(()) } @@ -96,13 +87,17 @@ where /// When a task is processed, the result of the process is pushed to its event list. The /// `handle_batch_result` make sure that the new state is saved to the store. /// The tasks are then removed from the processing queue. - async fn handle_batch_result(&self, mut batch: Batch) -> Result<()> { + async fn handle_batch_result( + &self, + mut batch: Batch, + performer: Arc, + ) -> Result<()> { let mut scheduler = self.scheduler.write().await; - let tasks = scheduler.update_tasks(batch.tasks).await?; + let content = scheduler.update_tasks(batch.content).await?; scheduler.finish(); drop(scheduler); - batch.tasks = tasks; - self.performer.finish(&batch).await; + batch.content = content; + performer.finish(&batch).await; Ok(()) } } diff --git a/meilisearch-lib/src/update_file_store.rs b/meilisearch-lib/src/update_file_store.rs index ec355a56e..3a60dfe26 100644 --- a/meilisearch-lib/src/update_file_store.rs +++ b/meilisearch-lib/src/update_file_store.rs @@ -26,7 +26,7 @@ pub struct UpdateFile { #[error("Error while persisting update to disk: {0}")] pub struct UpdateFileStoreError(Box); -type Result = std::result::Result; +pub type Result = std::result::Result; macro_rules! into_update_store_error { ($($other:path),*) => { @@ -249,7 +249,7 @@ mod test { pub async fn delete(&self, uuid: Uuid) -> Result<()> { match self { MockUpdateFileStore::Real(s) => s.delete(uuid).await, - MockUpdateFileStore::Mock(_) => todo!(), + MockUpdateFileStore::Mock(mocker) => unsafe { mocker.get("delete").call(uuid) }, } } } diff --git a/meilisearch-error/Cargo.toml b/meilisearch-types/Cargo.toml similarity index 89% rename from meilisearch-error/Cargo.toml rename to meilisearch-types/Cargo.toml index 0d1bd1d3a..6949722e7 100644 --- a/meilisearch-error/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "meilisearch-error" -version = "0.27.2" +name = "meilisearch-types" +version = "0.28.0" authors = ["marin "] edition = "2021" diff --git a/meilisearch-error/src/lib.rs b/meilisearch-types/src/error.rs similarity index 93% rename from meilisearch-error/src/lib.rs rename to meilisearch-types/src/error.rs index 11613497c..56ac65f9e 100644 --- a/meilisearch-error/src/lib.rs +++ b/meilisearch-types/src/error.rs @@ -73,12 +73,12 @@ impl aweb::error::ResponseError for ResponseError { pub trait ErrorCode: std::error::Error { fn error_code(&self) -> Code; - /// returns the HTTP status code ascociated with the error + /// returns the HTTP status code associated with the error fn http_status(&self) -> StatusCode { self.error_code().http() } - /// returns the doc url ascociated with the error + /// returns the doc url associated with the error fn error_url(&self) -> String { self.error_code().url() } @@ -166,10 +166,14 @@ pub enum Code { InvalidApiKeyIndexes, InvalidApiKeyExpiresAt, InvalidApiKeyDescription, + InvalidApiKeyName, + InvalidApiKeyUid, + ImmutableField, + ApiKeyAlreadyExists, } impl Code { - /// ascociate a `Code` variant to the actual ErrCode + /// associate a `Code` variant to the actual ErrCode fn err_code(&self) -> ErrCode { use Code::*; @@ -272,13 +276,17 @@ impl Code { InvalidApiKeyDescription => { ErrCode::invalid("invalid_api_key_description", StatusCode::BAD_REQUEST) } + InvalidApiKeyName => ErrCode::invalid("invalid_api_key_name", StatusCode::BAD_REQUEST), + InvalidApiKeyUid => ErrCode::invalid("invalid_api_key_uid", StatusCode::BAD_REQUEST), + ApiKeyAlreadyExists => ErrCode::invalid("api_key_already_exists", StatusCode::CONFLICT), + ImmutableField => ErrCode::invalid("immutable_field", StatusCode::BAD_REQUEST), InvalidMinWordLengthForTypo => { ErrCode::invalid("invalid_min_word_length_for_typo", StatusCode::BAD_REQUEST) } } } - /// return the HTTP status code ascociated with the `Code` + /// return the HTTP status code associated with the `Code` fn http(&self) -> StatusCode { self.err_code().status_code } @@ -293,7 +301,7 @@ impl Code { self.err_code().error_type.to_string() } - /// return the doc url ascociated with the error + /// return the doc url associated with the error fn url(&self) -> String { format!("https://docs.meilisearch.com/errors#{}", self.name()) } diff --git a/meilisearch-types/src/index_uid.rs b/meilisearch-types/src/index_uid.rs new file mode 100644 index 000000000..de453572b --- /dev/null +++ b/meilisearch-types/src/index_uid.rs @@ -0,0 +1,85 @@ +use serde::{Deserialize, Serialize}; +use std::error::Error; +use std::fmt; +use std::str::FromStr; + +/// An index uid is composed of only ascii alphanumeric characters, - and _, between 1 and 400 +/// bytes long +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[cfg_attr(feature = "test-traits", derive(proptest_derive::Arbitrary))] +pub struct IndexUid( + #[cfg_attr(feature = "test-traits", proptest(regex("[a-zA-Z0-9_-]{1,400}")))] String, +); + +impl IndexUid { + pub fn new_unchecked(s: impl AsRef) -> Self { + Self(s.as_ref().to_string()) + } + + pub fn into_inner(self) -> String { + self.0 + } + + /// Return a reference over the inner str. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::ops::Deref for IndexUid { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl TryFrom for IndexUid { + type Error = IndexUidFormatError; + + fn try_from(uid: String) -> Result { + if !uid + .chars() + .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') + || uid.is_empty() + || uid.len() > 400 + { + Err(IndexUidFormatError { invalid_uid: uid }) + } else { + Ok(IndexUid(uid)) + } + } +} + +impl FromStr for IndexUid { + type Err = IndexUidFormatError; + + fn from_str(uid: &str) -> Result { + uid.to_string().try_into() + } +} + +impl From for String { + fn from(uid: IndexUid) -> Self { + uid.into_inner() + } +} + +#[derive(Debug)] +pub struct IndexUidFormatError { + pub invalid_uid: String, +} + +impl fmt::Display for IndexUidFormatError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "invalid index uid `{}`, the uid must be an integer \ + or a string containing only alphanumeric characters \ + a-z A-Z 0-9, hyphens - and underscores _.", + self.invalid_uid, + ) + } +} + +impl Error for IndexUidFormatError {} diff --git a/meilisearch-types/src/lib.rs b/meilisearch-types/src/lib.rs new file mode 100644 index 000000000..2d685c2dc --- /dev/null +++ b/meilisearch-types/src/lib.rs @@ -0,0 +1,3 @@ +pub mod error; +pub mod index_uid; +pub mod star_or; diff --git a/meilisearch-types/src/star_or.rs b/meilisearch-types/src/star_or.rs new file mode 100644 index 000000000..02c9c3524 --- /dev/null +++ b/meilisearch-types/src/star_or.rs @@ -0,0 +1,138 @@ +use serde::de::Visitor; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt::{Display, Formatter}; +use std::marker::PhantomData; +use std::ops::Deref; +use std::str::FromStr; + +/// A type that tries to match either a star (*) or +/// any other thing that implements `FromStr`. +#[derive(Debug)] +pub enum StarOr { + Star, + Other(T), +} + +impl FromStr for StarOr { + type Err = T::Err; + + fn from_str(s: &str) -> Result { + if s.trim() == "*" { + Ok(StarOr::Star) + } else { + T::from_str(s).map(StarOr::Other) + } + } +} + +impl> Deref for StarOr { + type Target = str; + + fn deref(&self) -> &Self::Target { + match self { + Self::Star => "*", + Self::Other(t) => t.deref(), + } + } +} + +impl> From> for String { + fn from(s: StarOr) -> Self { + match s { + StarOr::Star => "*".to_string(), + StarOr::Other(t) => t.into(), + } + } +} + +impl PartialEq for StarOr { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Star, Self::Star) => true, + (Self::Other(left), Self::Other(right)) if left.eq(right) => true, + _ => false, + } + } +} + +impl Eq for StarOr {} + +impl<'de, T, E> Deserialize<'de> for StarOr +where + T: FromStr, + E: Display, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + /// Serde can't differentiate between `StarOr::Star` and `StarOr::Other` without a tag. + /// Simply using `#[serde(untagged)]` + `#[serde(rename="*")]` will lead to attempting to + /// deserialize everything as a `StarOr::Other`, including "*". + /// [`#[serde(other)]`](https://serde.rs/variant-attrs.html#other) might have helped but is + /// not supported on untagged enums. + struct StarOrVisitor(PhantomData); + + impl<'de, T, FE> Visitor<'de> for StarOrVisitor + where + T: FromStr, + FE: Display, + { + type Value = StarOr; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + formatter.write_str("a string") + } + + fn visit_str(self, v: &str) -> Result + where + SE: serde::de::Error, + { + match v { + "*" => Ok(StarOr::Star), + v => { + let other = FromStr::from_str(v).map_err(|e: T::Err| { + SE::custom(format!("Invalid `other` value: {}", e)) + })?; + Ok(StarOr::Other(other)) + } + } + } + } + + deserializer.deserialize_str(StarOrVisitor(PhantomData)) + } +} + +impl Serialize for StarOr +where + T: Deref, +{ + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + StarOr::Star => serializer.serialize_str("*"), + StarOr::Other(other) => serializer.serialize_str(other.deref()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, Value}; + + #[test] + fn star_or_serde_roundtrip() { + fn roundtrip(content: Value, expected: StarOr) { + let deserialized: StarOr = serde_json::from_value(content.clone()).unwrap(); + assert_eq!(deserialized, expected); + assert_eq!(content, serde_json::to_value(deserialized).unwrap()); + } + + roundtrip(json!("products"), StarOr::Other("products".to_string())); + roundtrip(json!("*"), StarOr::Star); + } +} diff --git a/permissive-json-pointer/Cargo.toml b/permissive-json-pointer/Cargo.toml index b50f30f19..9e01b81ab 100644 --- a/permissive-json-pointer/Cargo.toml +++ b/permissive-json-pointer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "permissive-json-pointer" -version = "0.2.0" +version = "0.28.0" edition = "2021" description = "A permissive json pointer" readme = "README.md" diff --git a/permissive-json-pointer/src/lib.rs b/permissive-json-pointer/src/lib.rs index 56382beae..8f97ab2de 100644 --- a/permissive-json-pointer/src/lib.rs +++ b/permissive-json-pointer/src/lib.rs @@ -206,7 +206,7 @@ fn create_value(value: &Document, mut selectors: HashSet<&str>) -> Document { new_value } -fn create_array(array: &Vec, selectors: &HashSet<&str>) -> Vec { +fn create_array(array: &[Value], selectors: &HashSet<&str>) -> Vec { let mut res = Vec::new(); for value in array {