diff --git a/.github/workflows/publish-apt-brew-pkg.yml b/.github/workflows/publish-apt-brew-pkg.yml index f7ab8666b..043715224 100644 --- a/.github/workflows/publish-apt-brew-pkg.yml +++ b/.github/workflows/publish-apt-brew-pkg.yml @@ -35,7 +35,7 @@ jobs: - name: Build deb package run: cargo deb -p meilisearch -o target/debian/meilisearch.deb - name: Upload debian pkg to release - uses: svenstaro/upload-release-action@2.6.1 + uses: svenstaro/upload-release-action@2.7.0 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/debian/meilisearch.deb diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index c79176439..2372ce497 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -54,7 +54,7 @@ jobs: # No need to upload binaries for dry run (cron) - name: Upload binaries to release if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.6.1 + uses: svenstaro/upload-release-action@2.7.0 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/release/meilisearch @@ -87,7 +87,7 @@ jobs: # No need to upload binaries for dry run (cron) - name: Upload binaries to release if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.6.1 + uses: svenstaro/upload-release-action@2.7.0 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/release/${{ matrix.artifact_name }} @@ -121,7 +121,7 @@ jobs: - name: Upload the binary to release # No need to upload binaries for dry run (cron) if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.6.1 + uses: svenstaro/upload-release-action@2.7.0 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/${{ matrix.target }}/release/meilisearch @@ -183,7 +183,7 @@ jobs: - name: Upload the binary to release # No need to upload binaries for dry run (cron) if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.6.1 + uses: svenstaro/upload-release-action@2.7.0 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/${{ matrix.target }}/release/meilisearch diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index ed264592c..890b4c424 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -43,7 +43,7 @@ jobs: toolchain: nightly override: true - name: Cache dependencies - uses: Swatinem/rust-cache@v2.5.0 + uses: Swatinem/rust-cache@v2.5.1 - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: @@ -65,7 +65,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Cache dependencies - uses: Swatinem/rust-cache@v2.5.0 + uses: Swatinem/rust-cache@v2.5.1 - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: @@ -146,7 +146,7 @@ jobs: toolchain: stable override: true - name: Cache dependencies - uses: Swatinem/rust-cache@v2.5.0 + uses: Swatinem/rust-cache@v2.5.1 - name: Run tests in debug uses: actions-rs/cargo@v1 with: @@ -165,7 +165,7 @@ jobs: override: true components: clippy - name: Cache dependencies - uses: Swatinem/rust-cache@v2.5.0 + uses: Swatinem/rust-cache@v2.5.1 - name: Run cargo clippy uses: actions-rs/cargo@v1 with: @@ -184,7 +184,7 @@ jobs: override: true components: rustfmt - name: Cache dependencies - uses: Swatinem/rust-cache@v2.5.0 + uses: Swatinem/rust-cache@v2.5.1 - name: Run cargo fmt # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file. # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate diff --git a/Cargo.lock b/Cargo.lock index d4813d209..dd51e7b07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bytes", "futures-core", "futures-sink", @@ -47,7 +47,7 @@ dependencies = [ "actix-utils", "ahash 0.8.3", "base64 0.21.2", - "bitflags", + "bitflags 1.3.2", "brotli", "bytes", "bytestring", @@ -405,7 +405,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -416,7 +416,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -428,17 +428,6 @@ dependencies = [ "critical-section", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -527,6 +516,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "block-buffer" version = "0.10.4" @@ -608,7 +603,7 @@ checksum = "fdde5c9cd29ebd706ce1b35600920a33550e402fc998a2e53ad3b42c3c47a192" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -705,16 +700,15 @@ dependencies = [ [[package]] name = "charabia" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb49850f555eb71aa6fc6d4d79420e81f4d89fa56e0e9c0f6d19aace2f56c554" +checksum = "57aa1b4a8dda126c03ebf2f7e31d16cfc8781c2fe80dedd1a33459efc3e07578" dependencies = [ "aho-corasick", "cow-utils", "csv", "deunicode", "either", - "finl_unicode", "fst", "irg-kvariants", "jieba-rs", @@ -767,18 +761,6 @@ dependencies = [ "inout", ] -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "bitflags", - "clap_lex 0.2.4", - "indexmap", - "textwrap", -] - [[package]] name = "clap" version = "4.3.0" @@ -798,8 +780,8 @@ checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" dependencies = [ "anstream", "anstyle", - "bitflags", - "clap_lex 0.5.0", + "bitflags 1.3.2", + "clap_lex", "strsim", ] @@ -812,16 +794,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.18", -] - -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", + "syn 2.0.28", ] [[package]] @@ -929,19 +902,19 @@ dependencies = [ [[package]] name = "criterion" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", - "atty", "cast", "ciborium", - "clap 3.2.25", + "clap", "criterion-plot", + "is-terminal", "itertools", - "lazy_static", "num-traits", + "once_cell", "oorandom", "plotters", "rayon", @@ -1048,9 +1021,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" dependencies = [ "csv-core", "itoa", @@ -1224,12 +1197,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "dump" version = "1.3.0" @@ -1369,7 +1336,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -1385,6 +1352,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -1469,12 +1442,6 @@ dependencies = [ "nom_locate", ] -[[package]] -name = "finl_unicode" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" - [[package]] name = "flate2" version = "1.0.26" @@ -1570,7 +1537,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -1608,7 +1575,7 @@ name = "fuzzers" version = "1.3.0" dependencies = [ "arbitrary", - "clap 4.3.0", + "clap", "fastrand", "milli", "serde", @@ -1676,7 +1643,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "libgit2-sys", "log", @@ -1712,7 +1679,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1734,15 +1701,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "hashbrown" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" -dependencies = [ - "ahash 0.7.6", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -1752,6 +1710,12 @@ dependencies = [ "ahash 0.7.6", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "heapless" version = "0.7.16" @@ -1773,8 +1737,8 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "heed" -version = "0.12.5" -source = "git+https://github.com/meilisearch/heed?tag=v0.12.6#8c5b94225fc949c02bb7b900cc50ffaf6b584b1e" +version = "0.12.7" +source = "git+https://github.com/meilisearch/heed?tag=v0.12.7#061a5276b1f336f5f3302bee291e336041d88632" dependencies = [ "byteorder", "heed-traits", @@ -1791,12 +1755,12 @@ dependencies = [ [[package]] name = "heed-traits" version = "0.7.0" -source = "git+https://github.com/meilisearch/heed?tag=v0.12.6#8c5b94225fc949c02bb7b900cc50ffaf6b584b1e" +source = "git+https://github.com/meilisearch/heed?tag=v0.12.7#061a5276b1f336f5f3302bee291e336041d88632" [[package]] name = "heed-types" version = "0.7.2" -source = "git+https://github.com/meilisearch/heed?tag=v0.12.6#8c5b94225fc949c02bb7b900cc50ffaf6b584b1e" +source = "git+https://github.com/meilisearch/heed?tag=v0.12.7#061a5276b1f336f5f3302bee291e336041d88632" dependencies = [ "bincode", "heed-traits", @@ -1805,15 +1769,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -1844,22 +1799,6 @@ dependencies = [ "digest", ] -[[package]] -name = "hnsw" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b9740ebf8769ec4ad6762cc951ba18f39bba6dfbc2fbbe46285f7539af79752" -dependencies = [ - "ahash 0.7.6", - "hashbrown 0.11.2", - "libm", - "num-traits", - "rand_core", - "serde", - "smallvec", - "space", -] - [[package]] name = "http" version = "0.2.9" @@ -1995,6 +1934,16 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "inout" version = "0.1.3" @@ -2029,6 +1978,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "instant-distance" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c619cdaa30bb84088963968bee12a45ea5fbbf355f2c021bcd15589f5ca494a" +dependencies = [ + "num_cpus", + "ordered-float", + "parking_lot", + "rand", + "rayon", + "serde", + "serde-big-array", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -2059,13 +2023,12 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.1", - "io-lifetimes", - "rustix 0.37.19", + "rustix 0.38.4", "windows-sys 0.48.0", ] @@ -2162,9 +2125,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.144" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libgit2-sys" @@ -2208,9 +2171,9 @@ dependencies = [ [[package]] name = "lindera-cc-cedict-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bf79b29a90bcd22036e494d6cc9ac3abe9ab604b21f3258ba6dc1ce501801" +checksum = "2d2e8f2ca97ddf952fe340642511b9c14b373cb2eef711d526bb8ef2ca0969b8" dependencies = [ "anyhow", "bincode", @@ -2227,9 +2190,9 @@ dependencies = [ [[package]] name = "lindera-compress" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f2e99e67736352bbb6ed1c273643975822505067ca32194b0981040bc50527a" +checksum = "f72b460559bcbe8a9cee85ea4a5056133ed3abf373031191589236e656d65b59" dependencies = [ "anyhow", "flate2", @@ -2238,9 +2201,9 @@ dependencies = [ [[package]] name = "lindera-core" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3935e966409156f22cb4b334b21b0dce84b7aa1cad62214b466489d249c8e5" +checksum = "f586eb8a9393c32d5525e0e9336a3727bd1329674740097126f3b0bff8a1a1ea" dependencies = [ "anyhow", "bincode", @@ -2255,9 +2218,9 @@ dependencies = [ [[package]] name = "lindera-decompress" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7476406abb63c49d7f59c88b9b868ee8d2981495ea7e2c3ad129902f9916b3c6" +checksum = "1fb1facd8da698072fcc7338bd757730db53d59f313f44dd583fa03681dcc0e1" dependencies = [ "anyhow", "flate2", @@ -2266,9 +2229,9 @@ dependencies = [ [[package]] name = "lindera-dictionary" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808b7d2b3cabc25a4022526d484a4cfd1d5924dc76a26e0379707698841acef2" +checksum = "ec7be7410b1da7017a8948986b87af67082f605e9a716f0989790d795d677f0c" dependencies = [ "anyhow", "bincode", @@ -2286,9 +2249,9 @@ dependencies = [ [[package]] name = "lindera-ipadic-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f373a280958c930e5ee4a1e4db3a0ee0542afaf02d3b5cacb8cab4e298648e" +checksum = "705d07f8a45d04fd95149f7ad41a26d1f9e56c9c00402be6f9dd05e3d88b99c6" dependencies = [ "anyhow", "bincode", @@ -2307,9 +2270,9 @@ dependencies = [ [[package]] name = "lindera-ipadic-neologd-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92eff98e9ed1a7a412b91709c2343457a04ef02fa0c27c27e3a5892f5591eae9" +checksum = "633a93983ba13fba42328311a501091bd4a7aff0c94ae9eaa9d4733dd2b0468a" dependencies = [ "anyhow", "bincode", @@ -2328,9 +2291,9 @@ dependencies = [ [[package]] name = "lindera-ko-dic" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74c6d5bf7d8092bd6d10de7a5d74b70ea7cf234586235b0d6cdb903b05a6c9e2" +checksum = "a428e0d316b6c86f51bd919479692bc41ad840dba266ebc044663970f431ea18" dependencies = [ "bincode", "byteorder", @@ -2345,9 +2308,9 @@ dependencies = [ [[package]] name = "lindera-ko-dic-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a4add6d3c1e41ec9e2690d33e287d0223fb59a30ccee4980c23f31368cae1e" +checksum = "2a5288704c6b8a069c0a1705c38758e836497698b50453373ab3d56c6f9a7ef8" dependencies = [ "anyhow", "bincode", @@ -2365,9 +2328,9 @@ dependencies = [ [[package]] name = "lindera-tokenizer" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6a8acbd068019d1cdac7316f0dcb87f8e33ede2b13aa237f45114f9750afb8" +checksum = "106ba439b2e87529d9bbedbb88d69f635baba1195c26502b308f55a85885fc81" dependencies = [ "bincode", "byteorder", @@ -2380,9 +2343,9 @@ dependencies = [ [[package]] name = "lindera-unidic" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14abf0613d350b30d3b0406a33b1de8fa8d829f26516909421702174785991c8" +checksum = "3399b6dcfe1701333451d184ff3c677f433b320153427b146360c9e4bd8cb816" dependencies = [ "bincode", "byteorder", @@ -2397,9 +2360,9 @@ dependencies = [ [[package]] name = "lindera-unidic-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e204ed53d9bd63227d1e6a6c1f122ca039e00a8634ac32e7fb0281eeec8615c4" +checksum = "b698227fdaeac32289173ab389b990d4eb00a40cbc9912020f69a0c491dabf55" dependencies = [ "anyhow", "bincode", @@ -2433,6 +2396,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" + [[package]] name = "lmdb-rkv-sys" version = "0.15.1" @@ -2473,9 +2442,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "logging_timer" @@ -2514,7 +2483,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -2552,13 +2521,12 @@ dependencies = [ "assert-json-diff", "async-stream", "async-trait", - "atty", "brotli", "bstr", "byte-unit", "bytes", "cargo_toml", - "clap 4.3.0", + "clap", "crossbeam-channel", "deserr", "dump", @@ -2572,8 +2540,9 @@ dependencies = [ "hex", "http", "index-scheduler", - "indexmap", + "indexmap 1.9.3", "insta", + "is-terminal", "itertools", "jsonwebtoken", "lazy_static", @@ -2725,9 +2694,9 @@ dependencies = [ "geoutils", "grenad", "heed", - "hnsw", - "indexmap", + "indexmap 1.9.3", "insta", + "instant-distance", "itertools", "json-depth-checker", "levenshtein_automata", @@ -2752,7 +2721,6 @@ dependencies = [ "smallstr", "smallvec", "smartstring", - "space", "tempfile", "thiserror", "time", @@ -2913,9 +2881,9 @@ checksum = "f69e48cd7c8e5bb52a1da1287fdbfd877c32673176583ce664cd63b201aba385" [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oorandom" @@ -2932,12 +2900,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "os_str_bytes" -version = "6.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" - [[package]] name = "page_size" version = "0.4.2" @@ -3078,7 +3040,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -3236,7 +3198,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69" dependencies = [ - "bitflags", + "bitflags 1.3.2", "byteorder", "hex", "lazy_static", @@ -3297,9 +3259,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "5907a1b7c277254a8b15170f6e7c97cfa60ee7872a3217663bb81151e48184bb" dependencies = [ "proc-macro2", ] @@ -3372,7 +3334,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -3381,7 +3343,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -3523,7 +3485,7 @@ version = "0.36.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -3537,7 +3499,7 @@ version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -3545,6 +3507,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustix" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +dependencies = [ + "bitflags 2.3.3", + "errno", + "libc", + "linux-raw-sys 0.4.5", + "windows-sys 0.48.0", +] + [[package]] name = "rustls" version = "0.20.8" @@ -3647,13 +3622,22 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.163" +version = "1.0.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed" dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + [[package]] name = "serde-cs" version = "0.2.4" @@ -3665,22 +3649,22 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ - "indexmap", + "indexmap 2.0.0", "itoa", "ryu", "serde", @@ -3803,9 +3787,6 @@ name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" -dependencies = [ - "serde", -] [[package]] name = "smartstring" @@ -3828,16 +3809,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "space" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5ab9701ae895386d13db622abf411989deff7109b13b46b6173bb4ce5c1d123" -dependencies = [ - "doc-comment", - "num-traits", -] - [[package]] name = "spin" version = "0.5.2" @@ -3901,9 +3872,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -3988,30 +3959,24 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" - [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -4093,7 +4058,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -4169,7 +4134,7 @@ version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" dependencies = [ - "indexmap", + "indexmap 1.9.3", "serde", "serde_spanned", "toml_datetime", @@ -4418,7 +4383,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", "wasm-bindgen-shared", ] @@ -4452,7 +4417,7 @@ checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/README.md b/README.md index 97b3d0bbb..88621729d 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ You may also want to check out [Meilisearch 101](https://www.meilisearch.com/doc ## ⚑ Supercharge your Meilisearch experience -Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=engine&utm_medium=meilisearch). Get started with a 14-day free trial! No credit card required. +Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). No credit card required. ## 🧰 SDKs & integration tools @@ -87,7 +87,7 @@ Finally, for more in-depth information, refer to our articles explaining fundame Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want. -To request deletion of collected data, please write to us atΒ [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data. +To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data. If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation. diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index aa4229e05..f30478a9e 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -14,11 +14,11 @@ license.workspace = true anyhow = "1.0.70" csv = "1.2.1" milli = { path = "../milli" } -mimalloc = { version = "0.1.36", default-features = false } +mimalloc = { version = "0.1.37", default-features = false } serde_json = { version = "1.0.95", features = ["preserve_order"] } [dev-dependencies] -criterion = { version = "0.4.0", features = ["html_reports"] } +criterion = { version = "0.5.1", features = ["html_reports"] } rand = "0.8.5" rand_chacha = "0.3.1" roaring = "0.10.1" diff --git a/dump/src/lib.rs b/dump/src/lib.rs index 036de6010..fa3cfb49a 100644 --- a/dump/src/lib.rs +++ b/dump/src/lib.rs @@ -210,6 +210,7 @@ pub(crate) mod test { use big_s::S; use maplit::{btreemap, btreeset}; use meilisearch_types::facet_values_sort::FacetValuesSort; + use meilisearch_types::features::RuntimeTogglableFeatures; use meilisearch_types::index_uid_pattern::IndexUidPattern; use meilisearch_types::keys::{Action, Key}; use meilisearch_types::milli; @@ -421,7 +422,10 @@ pub(crate) mod test { } keys.flush().unwrap(); - // ========== TODO: create features here + // ========== experimental features + let features = create_test_features(); + + dump.create_experimental_features(features).unwrap(); // create the dump let mut file = tempfile::tempfile().unwrap(); @@ -431,6 +435,10 @@ pub(crate) mod test { file } + fn create_test_features() -> RuntimeTogglableFeatures { + RuntimeTogglableFeatures { vector_store: true, ..Default::default() } + } + #[test] fn test_creating_and_read_dump() { let mut file = create_test_dump(); @@ -475,5 +483,9 @@ pub(crate) mod test { for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) { assert_eq!(key.unwrap(), expected); } + + // ==== checking the features + let expected = create_test_features(); + assert_eq!(dump.features().unwrap().unwrap(), expected); } } diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index 0899579f8..af02888d2 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -195,8 +195,53 @@ pub(crate) mod test { use meili_snap::insta; use super::*; + use crate::reader::v6::RuntimeTogglableFeatures; - // TODO: add `features` to tests + #[test] + fn import_dump_v6_experimental() { + let dump = File::open("tests/assets/v6-with-experimental.dump").unwrap(); + let mut dump = DumpReader::open(dump).unwrap(); + + // top level infos + insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00"); + insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None"); + + // tasks + let tasks = dump.tasks().unwrap().collect::>>().unwrap(); + let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"d45cd8571703e58ae53c7bd7ce3f5c22"); + assert_eq!(update_files.len(), 2); + assert!(update_files[0].is_none()); // the dump creation + assert!(update_files[1].is_none()); // the processed document addition + + // keys + let keys = dump.keys().unwrap().collect::>>().unwrap(); + meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"13c2da155e9729c2344688cab29af71d"); + + // indexes + let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); + // the index are not ordered in any way by default + indexes.sort_by_key(|index| index.metadata().uid.to_string()); + + let mut test = indexes.pop().unwrap(); + assert!(indexes.is_empty()); + + insta::assert_json_snapshot!(test.metadata(), @r###" + { + "uid": "test", + "primaryKey": "id", + "createdAt": "2023-07-06T07:07:41.364694Z", + "updatedAt": "2023-07-06T07:07:41.396114Z" + } + "###); + + assert_eq!(test.documents().unwrap().count(), 1); + + assert_eq!( + dump.features().unwrap().unwrap(), + RuntimeTogglableFeatures { vector_store: true, ..Default::default() } + ); + } #[test] fn import_dump_v5() { @@ -274,6 +319,8 @@ pub(crate) mod test { let documents = spells.documents().unwrap().collect::>>().unwrap(); assert_eq!(documents.len(), 10); meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce"); + + assert_eq!(dump.features().unwrap(), None); } #[test] diff --git a/dump/src/writer.rs b/dump/src/writer.rs index 695610f78..3c8126876 100644 --- a/dump/src/writer.rs +++ b/dump/src/writer.rs @@ -292,6 +292,7 @@ pub(crate) mod test { β”‚ β”œ---- update_files/ β”‚ β”‚ β””---- 1.jsonl β”‚ β””---- queue.jsonl + β”œ---- experimental-features.json β”œ---- instance_uid.uuid β”œ---- keys.jsonl β””---- metadata.json diff --git a/dump/tests/assets/v6-with-experimental.dump b/dump/tests/assets/v6-with-experimental.dump new file mode 100644 index 000000000..ac6833dc3 Binary files /dev/null and b/dump/tests/assets/v6-with-experimental.dump differ diff --git a/filter-parser/src/lib.rs b/filter-parser/src/lib.rs index 0ae8a1532..11ffbb148 100644 --- a/filter-parser/src/lib.rs +++ b/filter-parser/src/lib.rs @@ -472,6 +472,77 @@ pub fn parse_filter(input: Span) -> IResult { terminated(|input| parse_expression(input, 0), eof)(input) } +impl<'a> std::fmt::Display for FilterCondition<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FilterCondition::Not(filter) => { + write!(f, "NOT ({filter})") + } + FilterCondition::Condition { fid, op } => { + write!(f, "{fid} {op}") + } + FilterCondition::In { fid, els } => { + write!(f, "{fid} IN[")?; + for el in els { + write!(f, "{el}, ")?; + } + write!(f, "]") + } + FilterCondition::Or(els) => { + write!(f, "OR[")?; + for el in els { + write!(f, "{el}, ")?; + } + write!(f, "]") + } + FilterCondition::And(els) => { + write!(f, "AND[")?; + for el in els { + write!(f, "{el}, ")?; + } + write!(f, "]") + } + FilterCondition::GeoLowerThan { point, radius } => { + write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius) + } + FilterCondition::GeoBoundingBox { + top_right_point: top_left_point, + bottom_left_point: bottom_right_point, + } => { + write!( + f, + "_geoBoundingBox([{}, {}], [{}, {}])", + top_left_point[0], + top_left_point[1], + bottom_right_point[0], + bottom_right_point[1] + ) + } + } + } +} +impl<'a> std::fmt::Display for Condition<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Condition::GreaterThan(token) => write!(f, "> {token}"), + Condition::GreaterThanOrEqual(token) => write!(f, ">= {token}"), + Condition::Equal(token) => write!(f, "= {token}"), + Condition::NotEqual(token) => write!(f, "!= {token}"), + Condition::Null => write!(f, "IS NULL"), + Condition::Empty => write!(f, "IS EMPTY"), + Condition::Exists => write!(f, "EXISTS"), + Condition::LowerThan(token) => write!(f, "< {token}"), + Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"), + Condition::Between { from, to } => write!(f, "{from} TO {to}"), + } + } +} +impl<'a> std::fmt::Display for Token<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{{{}}}", self.value()) + } +} + #[cfg(test)] pub mod tests { use super::*; @@ -852,74 +923,3 @@ pub mod tests { assert_eq!(token.value(), s); } } - -impl<'a> std::fmt::Display for FilterCondition<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - FilterCondition::Not(filter) => { - write!(f, "NOT ({filter})") - } - FilterCondition::Condition { fid, op } => { - write!(f, "{fid} {op}") - } - FilterCondition::In { fid, els } => { - write!(f, "{fid} IN[")?; - for el in els { - write!(f, "{el}, ")?; - } - write!(f, "]") - } - FilterCondition::Or(els) => { - write!(f, "OR[")?; - for el in els { - write!(f, "{el}, ")?; - } - write!(f, "]") - } - FilterCondition::And(els) => { - write!(f, "AND[")?; - for el in els { - write!(f, "{el}, ")?; - } - write!(f, "]") - } - FilterCondition::GeoLowerThan { point, radius } => { - write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius) - } - FilterCondition::GeoBoundingBox { - top_right_point: top_left_point, - bottom_left_point: bottom_right_point, - } => { - write!( - f, - "_geoBoundingBox([{}, {}], [{}, {}])", - top_left_point[0], - top_left_point[1], - bottom_right_point[0], - bottom_right_point[1] - ) - } - } - } -} -impl<'a> std::fmt::Display for Condition<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Condition::GreaterThan(token) => write!(f, "> {token}"), - Condition::GreaterThanOrEqual(token) => write!(f, ">= {token}"), - Condition::Equal(token) => write!(f, "= {token}"), - Condition::NotEqual(token) => write!(f, "!= {token}"), - Condition::Null => write!(f, "IS NULL"), - Condition::Empty => write!(f, "IS EMPTY"), - Condition::Exists => write!(f, "EXISTS"), - Condition::LowerThan(token) => write!(f, "< {token}"), - Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"), - Condition::Between { from, to } => write!(f, "{from} TO {to}"), - } - } -} -impl<'a> std::fmt::Display for Token<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{{{}}}", self.value()) - } -} diff --git a/flatten-serde-json/Cargo.toml b/flatten-serde-json/Cargo.toml index 1ad00b0b3..7b498ec4f 100644 --- a/flatten-serde-json/Cargo.toml +++ b/flatten-serde-json/Cargo.toml @@ -16,7 +16,7 @@ license.workspace = true serde_json = "1.0" [dev-dependencies] -criterion = { version = "0.4.0", features = ["html_reports"] } +criterion = { version = "0.5.1", features = ["html_reports"] } [[bench]] name = "benchmarks" diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index fc5a8a0f9..b7b8685aa 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -138,6 +138,12 @@ impl Query { index_vec.push(index_uid); Self { index_uids: Some(index_vec), ..self } } + + // Removes the `from` and `limit` restrictions from the query. + // Useful to get the total number of tasks matching a filter. + pub fn without_limits(self) -> Self { + Query { limit: None, from: None, ..self } + } } #[derive(Debug, Clone)] @@ -807,6 +813,11 @@ impl IndexScheduler { Ok(res) } + // Return true if there is at least one task that is processing. + pub fn is_task_processing(&self) -> Result { + Ok(!self.processing_tasks.read().unwrap().processing.is_empty()) + } + /// Return true iff there is at least one task associated with this index /// that is processing. pub fn is_index_processing(&self, index: &str) -> Result { @@ -817,7 +828,8 @@ impl IndexScheduler { Ok(nbr_index_processing_tasks > 0) } - /// Return the task ids matching the query from the user's point of view. + /// Return the task ids matching the query along with the total number of tasks + /// by ignoring the from and limit parameters from the user's point of view. /// /// There are two differences between an internal query and a query executed by /// the user. @@ -830,7 +842,13 @@ impl IndexScheduler { rtxn: &RoTxn, query: &Query, filters: &meilisearch_auth::AuthFilter, - ) -> Result { + ) -> Result<(RoaringBitmap, u64)> { + // compute all tasks matching the filter by ignoring the limits, to find the number of tasks matching + // the filter. + // As this causes us to compute the filter twice it is slightly inefficient, but doing it this way spares + // us from modifying the underlying implementation, and the performance remains sufficient. + // Should this change, we would modify `get_task_ids` to directly return the number of matching tasks. + let total_tasks = self.get_task_ids(rtxn, &query.clone().without_limits())?; let mut tasks = self.get_task_ids(rtxn, query)?; // If the query contains a list of index uid or there is a finite list of authorized indexes, @@ -853,10 +871,11 @@ impl IndexScheduler { } } - Ok(tasks) + Ok((tasks, total_tasks.len())) } - /// Return the tasks matching the query from the user's point of view. + /// Return the tasks matching the query from the user's point of view along + /// with the total number of tasks matching the query, ignoring from and limit. /// /// There are two differences between an internal query and a query executed by /// the user. @@ -868,11 +887,10 @@ impl IndexScheduler { &self, query: Query, filters: &meilisearch_auth::AuthFilter, - ) -> Result> { + ) -> Result<(Vec, u64)> { let rtxn = self.env.read_txn()?; - let tasks = self.get_task_ids_from_authorized_indexes(&rtxn, &query, filters)?; - + let (tasks, total) = self.get_task_ids_from_authorized_indexes(&rtxn, &query, filters)?; let tasks = self.get_existing_tasks( &rtxn, tasks.into_iter().rev().take(query.limit.unwrap_or(u32::MAX) as usize), @@ -883,16 +901,19 @@ impl IndexScheduler { let ret = tasks.into_iter(); if processing.is_empty() { - Ok(ret.collect()) + Ok((ret.collect(), total)) } else { - Ok(ret - .map(|task| match processing.contains(task.uid) { - true => { + Ok(( + ret.map(|task| { + if processing.contains(task.uid) { Task { status: Status::Processing, started_at: Some(started_at), ..task } + } else { + task } - false => task, }) - .collect()) + .collect(), + total, + )) } } @@ -1835,6 +1856,17 @@ mod tests { snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_third_task"); } + #[test] + fn test_task_is_processing() { + let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); + + index_scheduler.register(index_creation_task("index_a", "id")).unwrap(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task"); + + handle.advance_till([Start, BatchCreated]); + assert!(index_scheduler.is_task_processing().unwrap()); + } + /// We send a lot of tasks but notify the tasks scheduler only once as /// we send them very fast, we must make sure that they are all processed. #[test] @@ -2767,43 +2799,43 @@ mod tests { let rtxn = index_scheduler.env.read_txn().unwrap(); let query = Query { limit: Some(0), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[]"); let query = Query { limit: Some(1), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[2,]"); let query = Query { limit: Some(2), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[1,2,]"); let query = Query { from: Some(1), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,]"); let query = Query { from: Some(2), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,2,]"); let query = Query { from: Some(1), limit: Some(1), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[1,]"); let query = Query { from: Some(1), limit: Some(2), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,]"); @@ -2830,13 +2862,13 @@ mod tests { let rtxn = index_scheduler.env.read_txn().unwrap(); let query = Query { statuses: Some(vec![Status::Processing]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,]"); // only the processing tasks in the first tick let query = Query { statuses: Some(vec![Status::Enqueued]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[1,2,]"); // only the enqueued tasks in the first tick @@ -2845,7 +2877,7 @@ mod tests { statuses: Some(vec![Status::Enqueued, Status::Processing]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,2,]"); // both enqueued and processing tasks in the first tick @@ -2855,7 +2887,7 @@ mod tests { after_started_at: Some(start_time), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both enqueued and processing tasks in the first tick, but limited to those with a started_at @@ -2867,7 +2899,7 @@ mod tests { before_started_at: Some(start_time), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both enqueued and processing tasks in the first tick, but limited to those with a started_at @@ -2880,7 +2912,7 @@ mod tests { before_started_at: Some(start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both enqueued and processing tasks in the first tick, but limited to those with a started_at @@ -2907,7 +2939,7 @@ mod tests { before_started_at: Some(start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both succeeded and processing tasks in the first tick, but limited to those with a started_at @@ -2920,7 +2952,7 @@ mod tests { before_started_at: Some(start_time), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both succeeded and processing tasks in the first tick, but limited to those with a started_at @@ -2933,7 +2965,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both succeeded and processing tasks in the first tick, but limited to those with a started_at @@ -2953,7 +2985,7 @@ mod tests { let rtxn = index_scheduler.env.read_txn().unwrap(); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // we run the same query to verify that, and indeed find that the last task is matched @@ -2965,7 +2997,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // enqueued, succeeded, or processing tasks started after the second part of the test, should @@ -2977,7 +3009,7 @@ mod tests { // now the last task should have failed snapshot!(snapshot_index_scheduler(&index_scheduler), name: "end"); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // so running the last query should return nothing @@ -2989,7 +3021,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // but the same query on failed tasks should return the last task @@ -3001,7 +3033,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // but the same query on failed tasks should return the last task @@ -3014,7 +3046,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // same query but with an invalid uid @@ -3027,7 +3059,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // same query but with a valid uid @@ -3059,14 +3091,14 @@ mod tests { let rtxn = index_scheduler.env.read_txn().unwrap(); let query = Query { index_uids: Some(vec!["catto".to_owned()]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // only the first task associated with catto is returned, the indexSwap tasks are excluded! snapshot!(snapshot_bitmap(&tasks), @"[0,]"); let query = Query { index_uids: Some(vec!["catto".to_owned()]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes( &rtxn, &query, @@ -3080,7 +3112,7 @@ mod tests { snapshot!(snapshot_bitmap(&tasks), @"[]"); let query = Query::default(); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes( &rtxn, &query, @@ -3094,7 +3126,7 @@ mod tests { snapshot!(snapshot_bitmap(&tasks), @"[1,]"); let query = Query::default(); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes( &rtxn, &query, @@ -3113,7 +3145,7 @@ mod tests { snapshot!(snapshot_bitmap(&tasks), @"[0,1,]"); let query = Query::default(); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // we asked for all the tasks with all index authorized -> all tasks returned @@ -3146,7 +3178,7 @@ mod tests { let rtxn = index_scheduler.read_txn().unwrap(); let query = Query { canceled_by: Some(vec![task_cancelation.uid]), ..Query::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // 0 is not returned because it was not canceled, 3 is not returned because it is the uid of the @@ -3154,7 +3186,7 @@ mod tests { snapshot!(snapshot_bitmap(&tasks), @"[1,2,]"); let query = Query { canceled_by: Some(vec![task_cancelation.uid]), ..Query::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes( &rtxn, &query, diff --git a/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap b/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap new file mode 100644 index 000000000..f17bfe38f --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap @@ -0,0 +1,36 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +---------------------------------------------------------------------- +### Status: +enqueued [0,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [0,] +---------------------------------------------------------------------- +### Index Tasks: +index_a [0,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + diff --git a/json-depth-checker/Cargo.toml b/json-depth-checker/Cargo.toml index 01b2d03a7..b8162357b 100644 --- a/json-depth-checker/Cargo.toml +++ b/json-depth-checker/Cargo.toml @@ -15,7 +15,7 @@ license.workspace = true serde_json = "1.0" [dev-dependencies] -criterion = "0.4.0" +criterion = "0.5.1" [[bench]] name = "depth" diff --git a/meili-snap/src/lib.rs b/meili-snap/src/lib.rs index dea0d750b..32d3385d9 100644 --- a/meili-snap/src/lib.rs +++ b/meili-snap/src/lib.rs @@ -199,6 +199,30 @@ macro_rules! snapshot { }; } +/// Create a string from the value by serializing it as Json, optionally +/// redacting some parts of it. +/// +/// The second argument to the macro can be an object expression for redaction. +/// It's in the form { selector => replacement }. For more information about redactions +/// refer to the redactions feature in the `insta` guide. +#[macro_export] +macro_rules! json_string { + ($value:expr, {$($k:expr => $v:expr),*$(,)?}) => { + { + let (_, snap) = meili_snap::insta::_prepare_snapshot_for_redaction!($value, {$($k => $v),*}, Json, File); + snap + } + }; + ($value:expr) => {{ + let value = meili_snap::insta::_macro_support::serialize_value( + &$value, + meili_snap::insta::_macro_support::SerializationFormat::Json, + meili_snap::insta::_macro_support::SnapshotLocation::File + ); + value + }}; +} + #[cfg(test)] mod tests { use crate as meili_snap; @@ -250,27 +274,3 @@ mod tests { } } } - -/// Create a string from the value by serializing it as Json, optionally -/// redacting some parts of it. -/// -/// The second argument to the macro can be an object expression for redaction. -/// It's in the form { selector => replacement }. For more information about redactions -/// refer to the redactions feature in the `insta` guide. -#[macro_export] -macro_rules! json_string { - ($value:expr, {$($k:expr => $v:expr),*$(,)?}) => { - { - let (_, snap) = meili_snap::insta::_prepare_snapshot_for_redaction!($value, {$($k => $v),*}, Json, File); - snap - } - }; - ($value:expr) => {{ - let value = meili_snap::insta::_macro_support::serialize_value( - &$value, - meili_snap::insta::_macro_support::SerializationFormat::Json, - meili_snap::insta::_macro_support::SnapshotLocation::File - ); - value - }}; -} diff --git a/meilisearch-types/src/features.rs b/meilisearch-types/src/features.rs index 6d02fc47b..f62300485 100644 --- a/meilisearch-types/src/features.rs +++ b/meilisearch-types/src/features.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] #[serde(rename_all = "camelCase", default)] pub struct RuntimeTogglableFeatures { pub score_details: bool, diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index 72c6e27e1..d33f53906 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -19,6 +19,7 @@ actix-http = { version = "3.3.1", default-features = false, features = [ "compress-gzip", "rustls", ] } +actix-utils = "3.0.1" actix-web = { version = "4.3.1", default-features = false, features = [ "macros", "compress-brotli", @@ -50,13 +51,14 @@ futures-util = "0.3.28" http = "0.2.9" index-scheduler = { path = "../index-scheduler" } indexmap = { version = "1.9.3", features = ["serde-1"] } +is-terminal = "0.4.8" itertools = "0.10.5" jsonwebtoken = "8.3.0" lazy_static = "1.4.0" log = "0.4.17" meilisearch-auth = { path = "../meilisearch-auth" } meilisearch-types = { path = "../meilisearch-types" } -mimalloc = { version = "0.1.36", default-features = false } +mimalloc = { version = "0.1.37", default-features = false } mime = "0.3.17" num_cpus = "1.15.0" obkv = "0.2.0" @@ -102,8 +104,6 @@ uuid = { version = "1.3.1", features = ["serde", "v4"] } walkdir = "2.3.3" yaup = "0.2.1" serde_urlencoded = "0.7.1" -actix-utils = "3.0.1" -atty = "0.2.14" termcolor = "1.2.0" [dev-dependencies] @@ -154,5 +154,5 @@ thai = ["meilisearch-types/thai"] greek = ["meilisearch-types/greek"] [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.7/build.zip" -sha1 = "28b45bf772c84f9a6e16bc1689b393bfce8da7d6" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.11/build.zip" +sha1 = "83cd44ed1e5f97ecb581dc9f958a63f4ccc982d9" diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 25aa20a9a..28e715852 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -574,6 +574,10 @@ pub struct SearchAggregator { filter_total_number_of_criteria: usize, used_syntax: HashMap, + // attributes_to_search_on + // every time a search is done using attributes_to_search_on + attributes_to_search_on_total_number_of_uses: usize, + // q // The maximum number of terms in a q request max_terms_number: usize, @@ -647,6 +651,11 @@ impl SearchAggregator { ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count(); } + // attributes_to_search_on + if let Some(_) = query.attributes_to_search_on { + ret.attributes_to_search_on_total_number_of_uses = 1; + } + if let Some(ref q) = query.q { ret.max_terms_number = q.split_whitespace().count(); } @@ -720,9 +729,18 @@ impl SearchAggregator { let used_syntax = self.used_syntax.entry(key).or_insert(0); *used_syntax = used_syntax.saturating_add(value); } + + // attributes_to_search_on + self.attributes_to_search_on_total_number_of_uses = self + .attributes_to_search_on_total_number_of_uses + .saturating_add(other.attributes_to_search_on_total_number_of_uses); + // q self.max_terms_number = self.max_terms_number.max(other.max_terms_number); + // vector + self.max_vector_size = self.max_vector_size.max(other.max_vector_size); + // pagination self.max_limit = self.max_limit.max(other.max_limit); self.max_offset = self.max_offset.max(other.max_offset); @@ -761,17 +779,17 @@ impl SearchAggregator { if self.total_received == 0 { None } else { - // the index of the 99th percentage of value - let percentile_99th = 0.99 * (self.total_succeeded as f64 - 1.) + 1.; // we get all the values in a sorted manner let time_spent = self.time_spent.into_sorted_vec(); + // the index of the 99th percentage of value + let percentile_99th = time_spent.len() * 99 / 100; // We are only interested by the slowest value of the 99th fastest results - let time_spent = time_spent.get(percentile_99th as usize); + let time_spent = time_spent.get(percentile_99th); let properties = json!({ "user-agent": self.user_agents, "requests": { - "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), + "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), "total_succeeded": self.total_succeeded, "total_failed": self.total_received.saturating_sub(self.total_succeeded), // just to be sure we never panics "total_received": self.total_received, @@ -786,9 +804,15 @@ impl SearchAggregator { "avg_criteria_number": format!("{:.2}", self.filter_sum_of_criteria_terms as f64 / self.filter_total_number_of_criteria as f64), "most_used_syntax": self.used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), }, + "attributes_to_search_on": { + "total_number_of_uses": self.attributes_to_search_on_total_number_of_uses, + }, "q": { "max_terms_number": self.max_terms_number, }, + "vector": { + "max_vector_size": self.max_vector_size, + }, "pagination": { "max_limit": self.max_limit, "max_offset": self.max_offset, @@ -843,6 +867,10 @@ pub struct MultiSearchAggregator { // sum of the number of search queries in the requests, use with total_received to compute an average total_search_count: usize, + // scoring + show_ranking_score: bool, + show_ranking_score_details: bool, + // context user_agents: HashSet, } @@ -856,6 +884,9 @@ impl MultiSearchAggregator { let distinct_indexes: HashSet<_> = query.iter().map(|query| query.index_uid.as_str()).collect(); + let show_ranking_score = query.iter().any(|query| query.show_ranking_score); + let show_ranking_score_details = query.iter().any(|query| query.show_ranking_score_details); + Self { timestamp, total_received: 1, @@ -863,6 +894,8 @@ impl MultiSearchAggregator { total_distinct_index_count: distinct_indexes.len(), total_single_index: if distinct_indexes.len() == 1 { 1 } else { 0 }, total_search_count: query.len(), + show_ranking_score, + show_ranking_score_details, user_agents, } } @@ -884,6 +917,9 @@ impl MultiSearchAggregator { this.total_distinct_index_count.saturating_add(other.total_distinct_index_count); let total_single_index = this.total_single_index.saturating_add(other.total_single_index); let total_search_count = this.total_search_count.saturating_add(other.total_search_count); + let show_ranking_score = this.show_ranking_score || other.show_ranking_score; + let show_ranking_score_details = + this.show_ranking_score_details || other.show_ranking_score_details; let mut user_agents = this.user_agents; for user_agent in other.user_agents.into_iter() { @@ -899,6 +935,8 @@ impl MultiSearchAggregator { total_single_index, total_search_count, user_agents, + show_ranking_score, + show_ranking_score_details, // do not add _ or ..Default::default() here }; @@ -925,6 +963,10 @@ impl MultiSearchAggregator { "searches": { "total_search_count": self.total_search_count, "avg_search_count": (self.total_search_count as f64) / (self.total_received as f64), + }, + "scoring": { + "show_ranking_score": self.show_ranking_score, + "show_ranking_score_details": self.show_ranking_score_details, } }); @@ -1145,6 +1187,7 @@ pub struct DocumentsDeletionAggregator { #[serde(rename = "user-agent")] user_agents: HashSet, + #[serde(rename = "requests.total_received")] total_received: usize, per_document_id: bool, clear_all: bool, @@ -1295,6 +1338,7 @@ pub struct HealthAggregator { #[serde(rename = "user-agent")] user_agents: HashSet, + #[serde(rename = "requests.total_received")] total_received: usize, } @@ -1345,7 +1389,7 @@ pub struct DocumentsFetchAggregator { #[serde(rename = "user-agent")] user_agents: HashSet, - #[serde(rename = "requests.max_limit")] + #[serde(rename = "requests.total_received")] total_received: usize, // a call on ../documents/:doc_id diff --git a/meilisearch/src/main.rs b/meilisearch/src/main.rs index 3ae3d27cb..a3905d451 100644 --- a/meilisearch/src/main.rs +++ b/meilisearch/src/main.rs @@ -1,5 +1,5 @@ use std::env; -use std::io::Write; +use std::io::{stderr, Write}; use std::path::PathBuf; use std::sync::Arc; @@ -7,6 +7,7 @@ use actix_web::http::KeepAlive; use actix_web::web::Data; use actix_web::HttpServer; use index_scheduler::IndexScheduler; +use is_terminal::IsTerminal; use meilisearch::analytics::Analytics; use meilisearch::{analytics, create_app, prototype_name, setup_meilisearch, Opt}; use meilisearch_auth::{generate_master_key, AuthController, MASTER_KEY_MIN_SIZE}; @@ -190,7 +191,7 @@ Anonymous telemetry:\t\"Enabled\"" } eprintln!(); - eprintln!("Check out Meilisearch Cloud!\thttps://cloud.meilisearch.com/login?utm_campaign=oss&utm_source=engine&utm_medium=cli"); + eprintln!("Check out Meilisearch Cloud!\thttps://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=engine&utm_medium=cli"); eprintln!("Documentation:\t\t\thttps://www.meilisearch.com/docs"); eprintln!("Source code:\t\t\thttps://github.com/meilisearch/meilisearch"); eprintln!("Discord:\t\t\thttps://discord.meilisearch.com"); @@ -201,8 +202,7 @@ const WARNING_BG_COLOR: Option = Some(Color::Ansi256(178)); const WARNING_FG_COLOR: Option = Some(Color::Ansi256(0)); fn print_master_key_too_short_warning() { - let choice = - if atty::is(atty::Stream::Stderr) { ColorChoice::Auto } else { ColorChoice::Never }; + let choice = if stderr().is_terminal() { ColorChoice::Auto } else { ColorChoice::Never }; let mut stderr = StandardStream::stderr(choice); stderr .set_color( @@ -227,8 +227,7 @@ fn print_master_key_too_short_warning() { } fn print_missing_master_key_warning() { - let choice = - if atty::is(atty::Stream::Stderr) { ColorChoice::Auto } else { ColorChoice::Never }; + let choice = if stderr().is_terminal() { ColorChoice::Auto } else { ColorChoice::Never }; let mut stderr = StandardStream::stderr(choice); stderr .set_color( diff --git a/meilisearch/src/metrics.rs b/meilisearch/src/metrics.rs index b3cbb4576..bfe704979 100644 --- a/meilisearch/src/metrics.rs +++ b/meilisearch/src/metrics.rs @@ -50,4 +50,10 @@ lazy_static! { &["kind", "value"] ) .expect("Can't create a metric"); + pub static ref MEILISEARCH_LAST_UPDATE: IntGauge = + register_int_gauge!(opts!("meilisearch_last_update", "Meilisearch Last Update")) + .expect("Can't create a metric"); + pub static ref MEILISEARCH_IS_INDEXING: IntGauge = + register_int_gauge!(opts!("meilisearch_is_indexing", "Meilisearch Is Indexing")) + .expect("Can't create a metric"); } diff --git a/meilisearch/src/routes/features.rs b/meilisearch/src/routes/features.rs index de5becfd1..a2822b4d4 100644 --- a/meilisearch/src/routes/features.rs +++ b/meilisearch/src/routes/features.rs @@ -64,7 +64,20 @@ async fn patch_features( vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store), }; - analytics.publish("Experimental features Updated".to_string(), json!(new_features), Some(&req)); + // explicitly destructure for analytics rather than using the `Serialize` implementation, because + // the it renames to camelCase, which we don't want for analytics. + // **Do not** ignore fields with `..` or `_` here, because we want to add them in the future. + let meilisearch_types::features::RuntimeTogglableFeatures { score_details, vector_store } = + new_features; + + analytics.publish( + "Experimental features Updated".to_string(), + json!({ + "score_details": score_details, + "vector_store": vector_store, + }), + Some(&req), + ); index_scheduler.put_runtime_features(new_features)?; Ok(HttpResponse::Ok().json(new_features)) } diff --git a/meilisearch/src/routes/indexes/search.rs b/meilisearch/src/routes/indexes/search.rs index 4d17adee6..3eabf61f3 100644 --- a/meilisearch/src/routes/indexes/search.rs +++ b/meilisearch/src/routes/indexes/search.rs @@ -35,7 +35,7 @@ pub struct SearchQueryGet { #[deserr(default, error = DeserrQueryParamError)] q: Option, #[deserr(default, error = DeserrQueryParamError)] - vector: Option>, + vector: Option>, #[deserr(default = Param(DEFAULT_SEARCH_OFFSET()), error = DeserrQueryParamError)] offset: Param, #[deserr(default = Param(DEFAULT_SEARCH_LIMIT()), error = DeserrQueryParamError)] @@ -88,7 +88,7 @@ impl From for SearchQuery { Self { q: other.q, - vector: other.vector, + vector: other.vector.map(CS::into_inner), offset: other.offset.0, limit: other.limit.0, page: other.page.as_deref().copied(), diff --git a/meilisearch/src/routes/indexes/settings.rs b/meilisearch/src/routes/indexes/settings.rs index eab08b895..3921b535e 100644 --- a/meilisearch/src/routes/indexes/settings.rs +++ b/meilisearch/src/routes/indexes/settings.rs @@ -5,6 +5,7 @@ use index_scheduler::IndexScheduler; use log::debug; use meilisearch_types::deserr::DeserrJsonError; use meilisearch_types::error::ResponseError; +use meilisearch_types::facet_values_sort::FacetValuesSort; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::settings::{settings, RankingRuleView, Settings, Unchecked}; use meilisearch_types::tasks::KindWithContent; @@ -628,10 +629,16 @@ pub async fn update_all( .as_ref() .set() .and_then(|s| s.max_values_per_facet.as_ref().set()), - "sort_facet_values_by": new_settings.faceting + "sort_facet_values_by_star_count": new_settings.faceting .as_ref() .set() - .and_then(|s| s.sort_facet_values_by.as_ref().set()), + .and_then(|s| { + s.sort_facet_values_by.as_ref().set().map(|s| s.iter().any(|(k, v)| k == "*" && v == &FacetValuesSort::Count)) + }), + "sort_facet_values_by_total": new_settings.faceting + .as_ref() + .set() + .and_then(|s| s.sort_facet_values_by.as_ref().set().map(|s| s.len())), }, "pagination": { "max_total_hits": new_settings.pagination diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index 492fec417..af1f2b536 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -49,6 +49,11 @@ pub async fn get_metrics( } } + if let Some(last_update) = response.last_update { + crate::metrics::MEILISEARCH_LAST_UPDATE.set(last_update.unix_timestamp()); + } + crate::metrics::MEILISEARCH_IS_INDEXING.set(index_scheduler.is_task_processing()? as i64); + let encoder = TextEncoder::new(); let mut buffer = vec![]; encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics"); diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index d14ddf48a..69bc44160 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -284,9 +284,6 @@ pub fn create_all_stats( used_database_size += index_scheduler.used_size()?; database_size += auth_controller.size()?; used_database_size += auth_controller.used_size()?; - let update_file_size = index_scheduler.compute_update_file_size()?; - database_size += update_file_size; - used_database_size += update_file_size; let stats = Stats { database_size, used_database_size, last_update: last_task, indexes }; Ok(stats) diff --git a/meilisearch/src/routes/tasks.rs b/meilisearch/src/routes/tasks.rs index 2713d0988..f7d4c44d7 100644 --- a/meilisearch/src/routes/tasks.rs +++ b/meilisearch/src/routes/tasks.rs @@ -325,7 +325,7 @@ async fn cancel_tasks( let query = params.into_query(); - let tasks = index_scheduler.get_task_ids_from_authorized_indexes( + let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes( &index_scheduler.read_txn()?, &query, index_scheduler.filters(), @@ -370,7 +370,7 @@ async fn delete_tasks( ); let query = params.into_query(); - let tasks = index_scheduler.get_task_ids_from_authorized_indexes( + let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes( &index_scheduler.read_txn()?, &query, index_scheduler.filters(), @@ -387,6 +387,7 @@ async fn delete_tasks( #[derive(Debug, Serialize)] pub struct AllTasks { results: Vec, + total: u64, limit: u32, from: Option, next: Option, @@ -406,23 +407,17 @@ async fn get_tasks( let limit = params.limit.0; let query = params.into_query(); - let mut tasks_results: Vec = index_scheduler - .get_tasks_from_authorized_indexes(query, index_scheduler.filters())? - .into_iter() - .map(|t| TaskView::from_task(&t)) - .collect(); + let filters = index_scheduler.filters(); + let (tasks, total) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?; + let mut results: Vec<_> = tasks.iter().map(TaskView::from_task).collect(); // If we were able to fetch the number +1 tasks we asked // it means that there is more to come. - let next = if tasks_results.len() == limit as usize { - tasks_results.pop().map(|t| t.uid) - } else { - None - }; + let next = if results.len() == limit as usize { results.pop().map(|t| t.uid) } else { None }; - let from = tasks_results.first().map(|t| t.uid); + let from = results.first().map(|t| t.uid); + let tasks = AllTasks { results, limit: limit.saturating_sub(1), total, from, next }; - let tasks = AllTasks { results: tasks_results, limit: limit.saturating_sub(1), from, next }; Ok(HttpResponse::Ok().json(tasks)) } @@ -444,10 +439,10 @@ async fn get_task( analytics.publish("Tasks Seen".to_string(), json!({ "per_task_uid": true }), Some(&req)); let query = index_scheduler::Query { uids: Some(vec![task_uid]), ..Query::default() }; + let filters = index_scheduler.filters(); + let (tasks, _) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?; - if let Some(task) = - index_scheduler.get_tasks_from_authorized_indexes(query, index_scheduler.filters())?.first() - { + if let Some(task) = tasks.first() { let task_view = TaskView::from_task(task); Ok(HttpResponse::Ok().json(task_view)) } else { diff --git a/meilisearch/tests/auth/authorization.rs b/meilisearch/tests/auth/authorization.rs index 58fba4481..5af214e5f 100644 --- a/meilisearch/tests/auth/authorization.rs +++ b/meilisearch/tests/auth/authorization.rs @@ -61,6 +61,8 @@ pub static AUTHORIZATIONS: Lazy hashset!{"keys.delete", "*"}, ("POST", "/keys") => hashset!{"keys.create", "*"}, ("GET", "/keys") => hashset!{"keys.get", "*"}, + ("GET", "/experimental-features") => hashset!{"experimental.get", "*"}, + ("PATCH", "/experimental-features") => hashset!{"experimental.update", "*"}, }; authorizations diff --git a/meilisearch/tests/common/server.rs b/meilisearch/tests/common/server.rs index 66b82eace..40d8e8366 100644 --- a/meilisearch/tests/common/server.rs +++ b/meilisearch/tests/common/server.rs @@ -189,6 +189,14 @@ impl Server { let url = format!("/tasks/{}", update_id); self.service.get(url).await } + + pub async fn get_features(&self) -> (Value, StatusCode) { + self.service.get("/experimental-features").await + } + + pub async fn set_features(&self, value: Value) -> (Value, StatusCode) { + self.service.patch("/experimental-features", value).await + } } pub fn default_settings(dir: impl AsRef) -> Opt { diff --git a/meilisearch/tests/dumps/mod.rs b/meilisearch/tests/dumps/mod.rs index 2feec996e..7e426cf54 100644 --- a/meilisearch/tests/dumps/mod.rs +++ b/meilisearch/tests/dumps/mod.rs @@ -85,7 +85,7 @@ async fn import_dump_v1_movie_raw() { snapshot!(code, @"200 OK"); assert_eq!( tasks, - json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.317060500S", "enqueuedAt": "2021-09-08T09:08:45.153219Z", "startedAt": "2021-09-08T09:08:45.3961665Z", "finishedAt": "2021-09-08T09:08:54.713227Z" }], "limit": 20, "from": 0, "next": null }) + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.317060500S", "enqueuedAt": "2021-09-08T09:08:45.153219Z", "startedAt": "2021-09-08T09:08:45.3961665Z", "finishedAt": "2021-09-08T09:08:54.713227Z" }], "total": 1, "limit": 20, "from": 0, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -245,7 +245,7 @@ async fn import_dump_v1_movie_with_settings() { snapshot!(code, @"200 OK"); assert_eq!( tasks, - json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["genres", "id", "overview", "poster", "release_date", "title"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT7.288826907S", "enqueuedAt": "2021-09-08T09:34:40.882977Z", "startedAt": "2021-09-08T09:34:40.883073093Z", "finishedAt": "2021-09-08T09:34:48.1719Z"}, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.090735774S", "enqueuedAt": "2021-09-08T09:34:16.036101Z", "startedAt": "2021-09-08T09:34:16.261191226Z", "finishedAt": "2021-09-08T09:34:25.351927Z" }], "limit": 20, "from": 1, "next": null }) + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["genres", "id", "overview", "poster", "release_date", "title"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT7.288826907S", "enqueuedAt": "2021-09-08T09:34:40.882977Z", "startedAt": "2021-09-08T09:34:40.883073093Z", "finishedAt": "2021-09-08T09:34:48.1719Z"}, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.090735774S", "enqueuedAt": "2021-09-08T09:34:16.036101Z", "startedAt": "2021-09-08T09:34:16.261191226Z", "finishedAt": "2021-09-08T09:34:25.351927Z" }], "total": 2, "limit": 20, "from": 1, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -523,7 +523,7 @@ async fn import_dump_v2_movie_raw() { snapshot!(code, @"200 OK"); assert_eq!( tasks, - json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "limit": 20, "from": 0, "next": null }) + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "total": 1, "limit": 20, "from": 0, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -667,7 +667,7 @@ async fn import_dump_v2_movie_with_settings() { snapshot!(code, @"200 OK"); assert_eq!( tasks, - json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "limit": 20, "from": 1, "next": null }) + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "total": 2, "limit": 20, "from": 1, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -942,7 +942,7 @@ async fn import_dump_v3_movie_raw() { snapshot!(code, @"200 OK"); assert_eq!( tasks, - json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "limit": 20, "from": 0, "next": null }) + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "total": 1, "limit": 20, "from": 0, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -1086,7 +1086,7 @@ async fn import_dump_v3_movie_with_settings() { snapshot!(code, @"200 OK"); assert_eq!( tasks, - json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "limit": 20, "from": 1, "next": null }) + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "total": 2, "limit": 20, "from": 1, "next": null }) ); // finally we're just going to check that we can["results"] still get a few documents by id @@ -1361,7 +1361,7 @@ async fn import_dump_v4_movie_raw() { snapshot!(code, @"200 OK"); assert_eq!( tasks, - json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "limit" : 20, "from": 0, "next": null }) + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "total": 1, "limit" : 20, "from": 0, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -1505,7 +1505,7 @@ async fn import_dump_v4_movie_with_settings() { snapshot!(code, @"200 OK"); assert_eq!( tasks, - json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "limit": 20, "from": 1, "next": null }) + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "total": 2, "limit": 20, "from": 1, "next": null }) ); // finally we're just going to check that we can still get a few documents by id diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap index 986b71132..d22c13b72 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:25.351927Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap index 986b71132..d22c13b72 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:25.351927Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap index 84a9c4fd5..beb25ce95 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap @@ -45,6 +45,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:26:57.319083Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap index fa8a9305b..add41439f 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 92, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap index 00d18a4f0..92613ce34 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap @@ -41,6 +41,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:40:28.669652Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap index 6a2fbe74f..bfef698e1 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 92, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap index 00d18a4f0..92613ce34 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap @@ -41,6 +41,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:40:28.669652Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap index 6a2fbe74f..bfef698e1 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 92, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap index 00d18a4f0..92613ce34 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap @@ -41,6 +41,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:40:28.669652Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap index 6a2fbe74f..bfef698e1 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 92, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap index 4ca8cbba0..b8c0d07b1 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2022-06-08T14:59:24.804443Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap index a5cee8ace..b31fb744f 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2022-06-08T14:59:29.997781Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap index 29b42313b..082cc97ee 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap @@ -19,6 +19,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 5, "limit": 1, "from": 4, "next": 3 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap index 9e63e3340..54e8f84bf 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap @@ -24,6 +24,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 2, "limit": 1, "from": 2, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap index 29b42313b..082cc97ee 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap @@ -19,6 +19,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 5, "limit": 1, "from": 4, "next": 3 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap index 29b42313b..082cc97ee 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap @@ -19,6 +19,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 5, "limit": 1, "from": 4, "next": 3 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap index 29b42313b..082cc97ee 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap @@ -19,6 +19,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 5, "limit": 1, "from": 4, "next": 3 diff --git a/meilisearch/tests/features/mod.rs b/meilisearch/tests/features/mod.rs new file mode 100644 index 000000000..045440f49 --- /dev/null +++ b/meilisearch/tests/features/mod.rs @@ -0,0 +1,109 @@ +use serde_json::json; + +use crate::common::Server; + +/// Feature name to test against. +/// This will have to be changed by a different one when that feature is stabilized. +/// All tests that need to set a feature can make use of this constant. +const FEATURE_NAME: &str = "vectorStore"; + +#[actix_rt::test] +async fn experimental_features() { + let server = Server::new().await; + + let (response, code) = server.get_features().await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": false + } + "###); + + let (response, code) = server.set_features(json!({FEATURE_NAME: true})).await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": true + } + "###); + + let (response, code) = server.get_features().await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": true + } + "###); + + // sending null does not change the value + let (response, code) = server.set_features(json!({FEATURE_NAME: null})).await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": true + } + "###); + + // not sending the field does not change the value + let (response, code) = server.set_features(json!({})).await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": true + } + "###); +} + +#[actix_rt::test] +async fn errors() { + let server = Server::new().await; + + // Sending a feature not in the list is an error + let (response, code) = server.set_features(json!({"NotAFeature": true})).await; + + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Unknown field `NotAFeature`: expected one of `scoreDetails`, `vectorStore`", + "code": "bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#bad_request" + } + "###); + + // The type must be a bool, not a number + let (response, code) = server.set_features(json!({FEATURE_NAME: 42})).await; + + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Invalid value type at `.vectorStore`: expected a boolean, but found a positive integer: `42`", + "code": "bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#bad_request" + } + "###); + + // The type must be a bool, not a string + let (response, code) = server.set_features(json!({FEATURE_NAME: "true"})).await; + + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Invalid value type at `.vectorStore`: expected a boolean, but found a string: `\"true\"`", + "code": "bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#bad_request" + } + "###); +} diff --git a/meilisearch/tests/integration.rs b/meilisearch/tests/integration.rs index 4383aea57..b6992791a 100644 --- a/meilisearch/tests/integration.rs +++ b/meilisearch/tests/integration.rs @@ -3,6 +3,7 @@ mod common; mod dashboard; mod documents; mod dumps; +mod features; mod index; mod search; mod settings; diff --git a/meilisearch/tests/search/errors.rs b/meilisearch/tests/search/errors.rs index 3d34de0fd..0745aa7d6 100644 --- a/meilisearch/tests/search/errors.rs +++ b/meilisearch/tests/search/errors.rs @@ -968,9 +968,12 @@ async fn sort_unset_ranking_rule() { async fn search_on_unknown_field() { let server = Server::new().await; let index = server.index("test"); + index.update_settings_searchable_attributes(json!(["id", "title"])).await; + index.wait_task(0).await; + let documents = DOCUMENTS.clone(); index.add_documents(documents, None).await; - index.wait_task(0).await; + index.wait_task(1).await; index .search( @@ -989,3 +992,49 @@ async fn search_on_unknown_field() { ) .await; } + +#[actix_rt::test] +async fn search_on_unknown_field_plus_joker() { + let server = Server::new().await; + let index = server.index("test"); + index.update_settings_searchable_attributes(json!(["id", "title"])).await; + index.wait_task(0).await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_task(1).await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["*", "unknown"]}), + |response, code| { + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r###" + { + "message": "Attribute `unknown` is not searchable. Available searchable attributes are: `id, title`.", + "code": "invalid_search_attributes_to_search_on", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_attributes_to_search_on" + } + "###); + }, + ) + .await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown", "*"]}), + |response, code| { + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r###" + { + "message": "Attribute `unknown` is not searchable. Available searchable attributes are: `id, title`.", + "code": "invalid_search_attributes_to_search_on", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_attributes_to_search_on" + } + "###); + }, + ) + .await; +} diff --git a/meilisearch/tests/search/facet_search.rs b/meilisearch/tests/search/facet_search.rs index 7628f2fed..446396856 100644 --- a/meilisearch/tests/search/facet_search.rs +++ b/meilisearch/tests/search/facet_search.rs @@ -1,3 +1,4 @@ +use meili_snap::snapshot; use once_cell::sync::Lazy; use serde_json::{json, Value}; @@ -56,6 +57,54 @@ async fn simple_facet_search() { assert_eq!(response["facetHits"].as_array().unwrap().len(), 1); } +#[actix_rt::test] +async fn advanced_facet_search() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.update_settings_filterable_attributes(json!(["genres"])).await; + index.update_settings_typo_tolerance(json!({ "enabled": false })).await; + index.add_documents(documents, None).await; + index.wait_task(2).await; + + let (response, code) = + index.facet_search(json!({"facetName": "genres", "facetQuery": "adventre"})).await; + + snapshot!(code, @"200 OK"); + snapshot!(response["facetHits"].as_array().unwrap().len(), @"0"); + + let (response, code) = + index.facet_search(json!({"facetName": "genres", "facetQuery": "Γ dventure"})).await; + + snapshot!(code, @"200 OK"); + snapshot!(response["facetHits"].as_array().unwrap().len(), @"1"); +} + +#[actix_rt::test] +async fn more_advanced_facet_search() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.update_settings_filterable_attributes(json!(["genres"])).await; + index.update_settings_typo_tolerance(json!({ "disableOnWords": ["adventre"] })).await; + index.add_documents(documents, None).await; + index.wait_task(2).await; + + let (response, code) = + index.facet_search(json!({"facetName": "genres", "facetQuery": "adventre"})).await; + + snapshot!(code, @"200 OK"); + snapshot!(response["facetHits"].as_array().unwrap().len(), @"0"); + + let (response, code) = + index.facet_search(json!({"facetName": "genres", "facetQuery": "adventure"})).await; + + snapshot!(code, @"200 OK"); + snapshot!(response["facetHits"].as_array().unwrap().len(), @"1"); +} + #[actix_rt::test] async fn non_filterable_facet_search_error() { let server = Server::new().await; diff --git a/meilisearch/tests/search/geo.rs b/meilisearch/tests/search/geo.rs new file mode 100644 index 000000000..4253bc210 --- /dev/null +++ b/meilisearch/tests/search/geo.rs @@ -0,0 +1,62 @@ +use once_cell::sync::Lazy; +use serde_json::{json, Value}; + +use crate::common::Server; + +pub(self) static DOCUMENTS: Lazy = Lazy::new(|| { + json!([ + { + "id": 1, + "name": "Taco Truck", + "address": "444 Salsa Street, Burritoville", + "type": "Mexican", + "rating": 9, + "_geo": { + "lat": 34.0522, + "lng": -118.2437 + } + }, + { + "id": 2, + "name": "La Bella Italia", + "address": "456 Elm Street, Townsville", + "type": "Italian", + "rating": 9, + "_geo": { + "lat": "45.4777599", + "lng": "9.1967508" + } + }, + { + "id": 3, + "name": "CrΓͺpe Truck", + "address": "2 Billig Avenue, Rouenville", + "type": "French", + "rating": 10 + } + ]) +}); + +#[actix_rt::test] +async fn geo_sort_with_geo_strings() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.update_settings_filterable_attributes(json!(["_geo"])).await; + index.update_settings_sortable_attributes(json!(["_geo"])).await; + index.add_documents(documents, None).await; + index.wait_task(2).await; + + index + .search( + json!({ + "filter": "_geoRadius(45.472735, 9.184019, 10000)", + "sort": ["_geoPoint(0.0, 0.0):asc"] + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + }, + ) + .await; +} diff --git a/meilisearch/tests/search/mod.rs b/meilisearch/tests/search/mod.rs index 97556ae2a..6fcc33309 100644 --- a/meilisearch/tests/search/mod.rs +++ b/meilisearch/tests/search/mod.rs @@ -4,6 +4,7 @@ mod errors; mod facet_search; mod formatted; +mod geo; mod multi; mod pagination; mod restrict_searchable; @@ -752,3 +753,354 @@ async fn faceting_max_values_per_facet() { ) .await; } + +#[actix_rt::test] +async fn experimental_feature_score_details() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + + index.add_documents(json!(documents), None).await; + index.wait_task(0).await; + + index + .search( + json!({ + "q": "train dragon", + "showRankingScoreDetails": true, + }), + |response, code| { + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Computing score details requires enabling the `score details` experimental feature. See https://github.com/meilisearch/product/discussions/674", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "###); + }, + ) + .await; + + let (response, code) = server.set_features(json!({"scoreDetails": true})).await; + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(response["scoreDetails"], @"true"); + + index + .search( + json!({ + "q": "train dragon", + "showRankingScoreDetails": true, + }), + |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "title": "How to Train Your Dragon: The Hidden World", + "id": "166428", + "_rankingScoreDetails": { + "words": { + "order": 0, + "matchingWords": 2, + "maxMatchingWords": 2, + "score": 1.0 + }, + "typo": { + "order": 1, + "typoCount": 0, + "maxTypoCount": 2, + "score": 1.0 + }, + "proximity": { + "order": 2, + "score": 0.875 + }, + "attribute": { + "order": 3, + "attributeRankingOrderScore": 1.0, + "queryWordDistanceScore": 0.8095238095238095, + "score": 0.9365079365079364 + }, + "exactness": { + "order": 4, + "matchType": "noExactMatch", + "matchingWords": 2, + "maxMatchingWords": 2, + "score": 0.3333333333333333 + } + } + } + ] + "###); + }, + ) + .await; +} + +#[actix_rt::test] +async fn experimental_feature_vector_store() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + + index.add_documents(json!(documents), None).await; + index.wait_task(0).await; + + let (response, code) = index + .search_post(json!({ + "vector": [1.0, 2.0, 3.0], + })) + .await; + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Passing `vector` as a query parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "###); + + let (response, code) = server.set_features(json!({"vectorStore": true})).await; + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(response["vectorStore"], @"true"); + + let (response, code) = index + .search_post(json!({ + "vector": [1.0, 2.0, 3.0], + })) + .await; + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @"[]"); +} + +#[cfg(feature = "default")] +#[actix_rt::test] +async fn camelcased_words() { + let server = Server::new().await; + let index = server.index("test"); + + // related to https://github.com/meilisearch/meilisearch/issues/3818 + let documents = json!([ + { "id": 0, "title": "DeLonghi" }, + { "id": 1, "title": "delonghi" }, + { "id": 2, "title": "TestAB" }, + { "id": 3, "title": "TestAb" }, + { "id": 4, "title": "testab" }, + ]); + index.add_documents(documents, None).await; + index.wait_task(0).await; + + index + .search(json!({"q": "deLonghi"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 0, + "title": "DeLonghi" + }, + { + "id": 1, + "title": "delonghi" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "dellonghi"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 0, + "title": "DeLonghi" + }, + { + "id": 1, + "title": "delonghi" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "testa"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "TestAb" + }, + { + "id": 4, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "testab"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "TestAb" + }, + { + "id": 4, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "TestaB"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "TestAb" + }, + { + "id": 4, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "Testab"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "TestAb" + }, + { + "id": 4, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "TestAb"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "TestAb" + }, + { + "id": 4, + "title": "testab" + } + ] + "###); + }) + .await; + + // with Typos + index + .search(json!({"q": "dellonghi"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 0, + "title": "DeLonghi" + }, + { + "id": 1, + "title": "delonghi" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "TetsAB"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "TestAb" + }, + { + "id": 4, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "TetsAB"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "TestAb" + }, + { + "id": 4, + "title": "testab" + } + ] + "###); + }) + .await; +} diff --git a/meilisearch/tests/search/restrict_searchable.rs b/meilisearch/tests/search/restrict_searchable.rs index f119acea5..219c747ed 100644 --- a/meilisearch/tests/search/restrict_searchable.rs +++ b/meilisearch/tests/search/restrict_searchable.rs @@ -49,6 +49,76 @@ async fn simple_search_on_title() { .await; } +#[actix_rt::test] +async fn search_no_searchable_attribute_set() { + let server = Server::new().await; + let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown"]}), + |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"0"); + }, + ) + .await; + + index.update_settings_searchable_attributes(json!(["*"])).await; + index.wait_task(1).await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown"]}), + |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"0"); + }, + ) + .await; + + index.update_settings_searchable_attributes(json!(["*"])).await; + index.wait_task(2).await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown", "title"]}), + |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"2"); + }, + ) + .await; +} + +#[actix_rt::test] +async fn search_on_all_attributes() { + let server = Server::new().await; + let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await; + + index + .search(json!({"q": "Captain Marvel", "attributesToSearchOn": ["*"]}), |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"3"); + }) + .await; +} + +#[actix_rt::test] +async fn search_on_all_attributes_restricted_set() { + let server = Server::new().await; + let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await; + index.update_settings_searchable_attributes(json!(["title"])).await; + index.wait_task(1).await; + + index + .search(json!({"q": "Captain Marvel", "attributesToSearchOn": ["*"]}), |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"2"); + }) + .await; +} + #[actix_rt::test] async fn simple_prefix_search_on_title() { let server = Server::new().await; @@ -240,7 +310,7 @@ async fn exactness_ranking_rule_order() { }, { "title": "Captain Marvel", - "desc": "CaptainMarvel", + "desc": "Captain the Marvel", "id": "2", }]), ) diff --git a/meilisearch/tests/swap_indexes/mod.rs b/meilisearch/tests/swap_indexes/mod.rs index 42d92e2af..4d01995b5 100644 --- a/meilisearch/tests/swap_indexes/mod.rs +++ b/meilisearch/tests/swap_indexes/mod.rs @@ -55,6 +55,7 @@ async fn swap_indexes() { "finishedAt": "[date]" } ], + "total": 2, "limit": 20, "from": 1, "next": null @@ -128,6 +129,7 @@ async fn swap_indexes() { "finishedAt": "[date]" } ], + "total": 3, "limit": 20, "from": 2, "next": null @@ -193,6 +195,7 @@ async fn swap_indexes() { "finishedAt": "[date]" } ], + "total": 5, "limit": 2, "from": 4, "next": 2 @@ -336,6 +339,7 @@ async fn swap_indexes() { "finishedAt": "[date]" } ], + "total": 6, "limit": 20, "from": 5, "next": null diff --git a/milli/Cargo.toml b/milli/Cargo.toml index 80b0a5f86..7ad15aa64 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -17,11 +17,11 @@ bincode = "1.3.3" bstr = "1.4.0" bytemuck = { version = "1.13.1", features = ["extern_crate_alloc"] } byteorder = "1.4.3" -charabia = { version = "0.8.1", default-features = false } +charabia = { version = "0.8.2", default-features = false } concat-arrays = "0.1.2" crossbeam-channel = "0.5.8" deserr = "0.5.0" -either = "1.8.1" +either = { version = "1.8.1", features = ["serde"] } flatten-serde-json = { path = "../flatten-serde-json" } fst = "0.4.7" fxhash = "0.2.1" @@ -29,12 +29,11 @@ geoutils = "0.5.1" grenad = { version = "0.4.4", default-features = false, features = [ "tempfile", ] } -heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.6", default-features = false, features = [ - "lmdb", - "sync-read-txn", +heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.7", default-features = false, features = [ + "lmdb", "read-txn-no-tls" ] } -hnsw = { version = "0.11.0", features = ["serde1"] } indexmap = { version = "1.9.3", features = ["serde"] } +instant-distance = { version = "0.6.1", features = ["with-serde"] } json-depth-checker = { path = "../json-depth-checker" } levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] } memmap2 = "0.5.10" @@ -48,7 +47,6 @@ rstar = { version = "0.10.0", features = ["serde"] } serde = { version = "1.0.160", features = ["derive"] } serde_json = { version = "1.0.95", features = ["preserve_order"] } slice-group-by = "0.3.0" -space = "0.17.0" smallstr = { version = "0.3.0", features = ["serde"] } smallvec = "1.10.0" smartstring = "1.0.1" @@ -76,7 +74,7 @@ logging_timer = "1.1.0" csv = "1.2.1" [dev-dependencies] -mimalloc = { version = "0.1.29", default-features = false } +mimalloc = { version = "0.1.37", default-features = false } big_s = "1.0.2" insta = "1.29.0" maplit = "1.0.2" @@ -84,7 +82,7 @@ md5 = "0.7.0" rand = { version = "0.8.5", features = ["small_rng"] } [features] -all-tokenizations = ["charabia/default"] +all-tokenizations = ["charabia/chinese", "charabia/hebrew", "charabia/japanese", "charabia/thai", "charabia/korean", "charabia/greek"] # Use POSIX semaphores instead of SysV semaphores in LMDB # For more information on this feature, see heed's Cargo.toml diff --git a/milli/src/distance.rs b/milli/src/distance.rs index c838e4bd4..e9e17e647 100644 --- a/milli/src/distance.rs +++ b/milli/src/distance.rs @@ -1,20 +1,36 @@ +use std::ops; + +use instant_distance::Point; use serde::{Deserialize, Serialize}; -use space::Metric; -#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] -pub struct DotProduct; +use crate::normalize_vector; -impl Metric> for DotProduct { - type Unit = u32; +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct NDotProductPoint(Vec); - // Following . - // - // Here is a playground that validate the ordering of the bit representation of floats in range 0.0..=1.0: - // - fn distance(&self, a: &Vec, b: &Vec) -> Self::Unit { - let dist = 1.0 - dot_product_similarity(a, b); +impl NDotProductPoint { + pub fn new(point: Vec) -> Self { + NDotProductPoint(normalize_vector(point)) + } + + pub fn into_inner(self) -> Vec { + self.0 + } +} + +impl ops::Deref for NDotProductPoint { + type Target = [f32]; + + fn deref(&self) -> &Self::Target { + self.0.as_slice() + } +} + +impl Point for NDotProductPoint { + fn distance(&self, other: &Self) -> f32 { + let dist = 1.0 - dot_product_similarity(&self.0, &other.0); debug_assert!(!dist.is_nan()); - dist.to_bits() + dist } } diff --git a/milli/src/heed_codec/beu16_str_codec.rs b/milli/src/heed_codec/beu16_str_codec.rs new file mode 100644 index 000000000..d1b85d47f --- /dev/null +++ b/milli/src/heed_codec/beu16_str_codec.rs @@ -0,0 +1,27 @@ +use std::borrow::Cow; +use std::convert::TryInto; +use std::str; + +pub struct BEU16StrCodec; + +impl<'a> heed::BytesDecode<'a> for BEU16StrCodec { + type DItem = (u16, &'a str); + + fn bytes_decode(bytes: &'a [u8]) -> Option { + let (n_bytes, str_bytes) = bytes.split_at(2); + let n = n_bytes.try_into().map(u16::from_be_bytes).ok()?; + let s = str::from_utf8(str_bytes).ok()?; + Some((n, s)) + } +} + +impl<'a> heed::BytesEncode<'a> for BEU16StrCodec { + type EItem = (u16, &'a str); + + fn bytes_encode((n, s): &Self::EItem) -> Option> { + let mut bytes = Vec::with_capacity(s.len() + 2); + bytes.extend_from_slice(&n.to_be_bytes()); + bytes.extend_from_slice(s.as_bytes()); + Some(Cow::Owned(bytes)) + } +} diff --git a/milli/src/heed_codec/mod.rs b/milli/src/heed_codec/mod.rs index 666f68e28..d04eaa644 100644 --- a/milli/src/heed_codec/mod.rs +++ b/milli/src/heed_codec/mod.rs @@ -1,3 +1,4 @@ +mod beu16_str_codec; mod beu32_str_codec; mod byte_slice_ref; pub mod facet; @@ -14,6 +15,7 @@ mod str_str_u8_codec; pub use byte_slice_ref::ByteSliceRefCodec; pub use str_ref::StrRefCodec; +pub use self::beu16_str_codec::BEU16StrCodec; pub use self::beu32_str_codec::BEU32StrCodec; pub use self::field_id_word_count_codec::FieldIdWordCountCodec; pub use self::fst_set_codec::FstSetCodec; diff --git a/milli/src/index.rs b/milli/src/index.rs index e1314896b..f3432628f 100644 --- a/milli/src/index.rs +++ b/milli/src/index.rs @@ -8,12 +8,11 @@ use charabia::{Language, Script}; use heed::flags::Flags; use heed::types::*; use heed::{CompactionOption, Database, PolyDatabase, RoTxn, RwTxn}; -use rand_pcg::Pcg32; use roaring::RoaringBitmap; use rstar::RTree; use time::OffsetDateTime; -use crate::distance::DotProduct; +use crate::distance::NDotProductPoint; use crate::error::{InternalError, UserError}; use crate::facet::FacetType; use crate::fields_ids_map::FieldsIdsMap; @@ -21,7 +20,9 @@ use crate::heed_codec::facet::{ FacetGroupKeyCodec, FacetGroupValueCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetStringCodec, FieldIdCodec, OrderedF64Codec, }; -use crate::heed_codec::{FstSetCodec, ScriptLanguageCodec, StrBEU16Codec, StrRefCodec}; +use crate::heed_codec::{ + BEU16StrCodec, FstSetCodec, ScriptLanguageCodec, StrBEU16Codec, StrRefCodec, +}; use crate::readable_slices::ReadableSlices; use crate::{ default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds, @@ -31,7 +32,7 @@ use crate::{ }; /// The HNSW data-structure that we serialize, fill and search in. -pub type Hnsw = hnsw::Hnsw, Pcg32, 12, 24>; +pub type Hnsw = instant_distance::Hnsw; pub const DEFAULT_MIN_WORD_LEN_ONE_TYPO: u8 = 5; pub const DEFAULT_MIN_WORD_LEN_TWO_TYPOS: u8 = 9; @@ -100,6 +101,7 @@ pub mod db_name { pub const FACET_ID_IS_NULL_DOCIDS: &str = "facet-id-is-null-docids"; pub const FACET_ID_IS_EMPTY_DOCIDS: &str = "facet-id-is-empty-docids"; pub const FACET_ID_STRING_DOCIDS: &str = "facet-id-string-docids"; + pub const FACET_ID_NORMALIZED_STRING_STRINGS: &str = "facet-id-normalized-string-strings"; pub const FACET_ID_STRING_FST: &str = "facet-id-string-fst"; pub const FIELD_ID_DOCID_FACET_F64S: &str = "field-id-docid-facet-f64s"; pub const FIELD_ID_DOCID_FACET_STRINGS: &str = "field-id-docid-facet-strings"; @@ -161,6 +163,8 @@ pub struct Index { pub facet_id_f64_docids: Database, FacetGroupValueCodec>, /// Maps the facet field id and ranges of strings with the docids that corresponds to them. pub facet_id_string_docids: Database, FacetGroupValueCodec>, + /// Maps the facet field id of the normalized-for-search string facets with their original versions. + pub facet_id_normalized_string_strings: Database>>, /// Maps the facet field id of the string facets with an FST containing all the facets values. pub facet_id_string_fst: Database, FstSetCodec>, @@ -185,7 +189,7 @@ impl Index { ) -> Result { use db_name::*; - options.max_dbs(24); + options.max_dbs(25); unsafe { options.flag(Flags::MdbAlwaysFreePages) }; let env = options.open(path)?; @@ -215,6 +219,8 @@ impl Index { let facet_id_f64_docids = env.create_database(&mut wtxn, Some(FACET_ID_F64_DOCIDS))?; let facet_id_string_docids = env.create_database(&mut wtxn, Some(FACET_ID_STRING_DOCIDS))?; + let facet_id_normalized_string_strings = + env.create_database(&mut wtxn, Some(FACET_ID_NORMALIZED_STRING_STRINGS))?; let facet_id_string_fst = env.create_database(&mut wtxn, Some(FACET_ID_STRING_FST))?; let facet_id_exists_docids = env.create_database(&mut wtxn, Some(FACET_ID_EXISTS_DOCIDS))?; @@ -250,6 +256,7 @@ impl Index { field_id_word_count_docids, facet_id_f64_docids, facet_id_string_docids, + facet_id_normalized_string_strings, facet_id_string_fst, facet_id_exists_docids, facet_id_is_null_docids, diff --git a/milli/src/lib.rs b/milli/src/lib.rs index 55b283931..3e5f63fd5 100644 --- a/milli/src/lib.rs +++ b/milli/src/lib.rs @@ -51,9 +51,10 @@ pub use self::error::{ pub use self::external_documents_ids::ExternalDocumentsIds; pub use self::fields_ids_map::FieldsIdsMap; pub use self::heed_codec::{ - BEU32StrCodec, BoRoaringBitmapCodec, BoRoaringBitmapLenCodec, CboRoaringBitmapCodec, - CboRoaringBitmapLenCodec, FieldIdWordCountCodec, ObkvCodec, RoaringBitmapCodec, - RoaringBitmapLenCodec, StrBEU32Codec, U8StrStrCodec, UncheckedU8StrStrCodec, + BEU16StrCodec, BEU32StrCodec, BoRoaringBitmapCodec, BoRoaringBitmapLenCodec, + CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, FieldIdWordCountCodec, ObkvCodec, + RoaringBitmapCodec, RoaringBitmapLenCodec, StrBEU32Codec, U8StrStrCodec, + UncheckedU8StrStrCodec, }; pub use self::index::Index; pub use self::search::{ diff --git a/milli/src/score_details.rs b/milli/src/score_details.rs index 37b486047..8fc998ae4 100644 --- a/milli/src/score_details.rs +++ b/milli/src/score_details.rs @@ -10,7 +10,7 @@ pub enum ScoreDetails { Fid(Rank), Position(Rank), ExactAttribute(ExactAttribute), - Exactness(Rank), + ExactWords(ExactWords), Sort(Sort), GeoSort(GeoSort), } @@ -28,7 +28,7 @@ impl ScoreDetails { ScoreDetails::Fid(details) => Some(*details), ScoreDetails::Position(details) => Some(*details), ScoreDetails::ExactAttribute(details) => Some(details.rank()), - ScoreDetails::Exactness(details) => Some(*details), + ScoreDetails::ExactWords(details) => Some(details.rank()), ScoreDetails::Sort(_) => None, ScoreDetails::GeoSort(_) => None, } @@ -84,7 +84,7 @@ impl ScoreDetails { // For now, fid is a virtual rule always followed by the "position" rule let fid_details = serde_json::json!({ "order": order, - "attribute_ranking_order_score": fid.local_score(), + "attributeRankingOrderScore": fid.local_score(), }); details_map.insert("attribute".into(), fid_details); order += 1; @@ -102,7 +102,7 @@ impl ScoreDetails { }; attribute_details - .insert("query_word_distance_score".into(), position.local_score().into()); + .insert("queryWordDistanceScore".into(), position.local_score().into()); let score = Rank::global_score([fid_details, *position].iter().copied()); attribute_details.insert("score".into(), score.into()); @@ -117,7 +117,7 @@ impl ScoreDetails { details_map.insert("exactness".into(), exactness_details); order += 1; } - ScoreDetails::Exactness(details) => { + ScoreDetails::ExactWords(details) => { // For now, exactness is a virtual rule always preceded by the "ExactAttribute" rule let exactness_details = details_map .get_mut("exactness") @@ -129,9 +129,16 @@ impl ScoreDetails { == &serde_json::json!(ExactAttribute::NoExactMatch) { let score = Rank::global_score( - [ExactAttribute::NoExactMatch.rank(), *details].iter().copied(), + [ExactAttribute::NoExactMatch.rank(), details.rank()].iter().copied(), ); - *exactness_details.get_mut("score").expect("missing score") = score.into(); + // tiny detail, but we want the score to be the last displayed field, + // so we're removing it here, adding the other fields, then adding the new score + exactness_details.remove("score"); + exactness_details + .insert("matchingWords".into(), details.matching_words.into()); + exactness_details + .insert("maxMatchingWords".into(), details.max_matching_words.into()); + exactness_details.insert("score".into(), score.into()); } // do not update the order since this was already done by exactAttribute } @@ -209,8 +216,34 @@ impl Words { Rank { rank: self.matching_words, max_rank: self.max_matching_words } } - pub(crate) fn from_rank(rank: Rank) -> Words { - Words { matching_words: rank.rank, max_matching_words: rank.max_rank } + pub(crate) fn from_rank(rank: Rank) -> Self { + Self { matching_words: rank.rank, max_matching_words: rank.max_rank } + } +} + +/// Structure that is super similar to [`Words`], but whose semantics is a bit distinct. +/// +/// In exactness, the number of matching words can actually be 0 with a non-zero score, +/// if no words from the query appear exactly in the document. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ExactWords { + pub matching_words: u32, + pub max_matching_words: u32, +} + +impl ExactWords { + pub fn rank(&self) -> Rank { + // 0 matching words means last rank (1) + Rank { rank: self.matching_words + 1, max_rank: self.max_matching_words + 1 } + } + + pub(crate) fn from_rank(rank: Rank) -> Self { + // last rank (1) means that 0 words from the query appear exactly in the document. + // first rank (max_rank) means that (max_rank - 1) words from the query appear exactly in the document. + Self { + matching_words: rank.rank.saturating_sub(1), + max_matching_words: rank.max_rank.saturating_sub(1), + } } } @@ -223,7 +256,7 @@ pub struct Typo { impl Typo { pub fn rank(&self) -> Rank { Rank { - rank: self.max_typo_count - self.typo_count + 1, + rank: (self.max_typo_count + 1).saturating_sub(self.typo_count), max_rank: (self.max_typo_count + 1), } } @@ -236,7 +269,10 @@ impl Typo { // rank + typo = max_rank // typo = max_rank - rank pub fn from_rank(rank: Rank) -> Typo { - Typo { typo_count: rank.max_rank - rank.rank, max_typo_count: rank.max_rank - 1 } + Typo { + typo_count: rank.max_rank.saturating_sub(rank.rank), + max_typo_count: rank.max_rank.saturating_sub(1), + } } } diff --git a/milli/src/search/mod.rs b/milli/src/search/mod.rs index 65e78caa9..e1ee37d79 100644 --- a/milli/src/search/mod.rs +++ b/milli/src/search/mod.rs @@ -1,5 +1,8 @@ use std::fmt; +use std::ops::ControlFlow; +use charabia::normalizer::NormalizerOption; +use charabia::Normalize; use fst::automaton::{Automaton, Str}; use fst::{IntoStreamer, Streamer}; use levenshtein_automata::{LevenshteinAutomatonBuilder as LevBuilder, DFA}; @@ -14,8 +17,8 @@ use crate::error::UserError; use crate::heed_codec::facet::{FacetGroupKey, FacetGroupValue}; use crate::score_details::{ScoreDetails, ScoringStrategy}; use crate::{ - execute_search, normalize_facet, AscDesc, DefaultSearchLogger, DocumentId, FieldId, Index, - Result, SearchContext, BEU16, + execute_search, AscDesc, DefaultSearchLogger, DocumentId, FieldId, Index, Result, + SearchContext, BEU16, }; // Building these factories is not free. @@ -301,29 +304,28 @@ impl<'a> SearchForFacetValues<'a> { match self.query.as_ref() { Some(query) => { - let query = normalize_facet(query); - let query = query.as_str(); + let options = NormalizerOption { lossy: true, ..Default::default() }; + let query = query.normalize(&options); + let query = query.as_ref(); + let authorize_typos = self.search_query.index.authorize_typos(rtxn)?; let field_authorizes_typos = !self.search_query.index.exact_attributes_ids(rtxn)?.contains(&fid); if authorize_typos && field_authorizes_typos { - let mut results = vec![]; - let exact_words_fst = self.search_query.index.exact_words(rtxn)?; if exact_words_fst.map_or(false, |fst| fst.contains(query)) { - let key = FacetGroupKey { field_id: fid, level: 0, left_bound: query }; - if let Some(FacetGroupValue { bitmap, .. }) = - index.facet_id_string_docids.get(rtxn, &key)? - { - let count = search_candidates.intersection_len(&bitmap); - if count != 0 { - let value = self - .one_original_value_of(fid, query, bitmap.min().unwrap())? - .unwrap_or_else(|| query.to_string()); - results.push(FacetValueHit { value, count }); - } + let mut results = vec![]; + if fst.contains(query) { + self.fetch_original_facets_using_normalized( + fid, + query, + query, + &search_candidates, + &mut results, + )?; } + Ok(results) } else { let one_typo = self.search_query.index.min_word_len_one_typo(rtxn)?; let two_typos = self.search_query.index.min_word_len_two_typos(rtxn)?; @@ -338,60 +340,41 @@ impl<'a> SearchForFacetValues<'a> { }; let mut stream = fst.search(automaton).into_stream(); - let mut length = 0; + let mut results = vec![]; while let Some(facet_value) = stream.next() { let value = std::str::from_utf8(facet_value)?; - let key = FacetGroupKey { field_id: fid, level: 0, left_bound: value }; - let docids = match index.facet_id_string_docids.get(rtxn, &key)? { - Some(FacetGroupValue { bitmap, .. }) => bitmap, - None => { - error!( - "the facet value is missing from the facet database: {key:?}" - ); - continue; - } - }; - let count = search_candidates.intersection_len(&docids); - if count != 0 { - let value = self - .one_original_value_of(fid, value, docids.min().unwrap())? - .unwrap_or_else(|| query.to_string()); - results.push(FacetValueHit { value, count }); - length += 1; - } - if length >= MAX_NUMBER_OF_FACETS { + if self + .fetch_original_facets_using_normalized( + fid, + value, + query, + &search_candidates, + &mut results, + )? + .is_break() + { break; } } - } - Ok(results) + Ok(results) + } } else { let automaton = Str::new(query).starts_with(); let mut stream = fst.search(automaton).into_stream(); let mut results = vec![]; - let mut length = 0; while let Some(facet_value) = stream.next() { let value = std::str::from_utf8(facet_value)?; - let key = FacetGroupKey { field_id: fid, level: 0, left_bound: value }; - let docids = match index.facet_id_string_docids.get(rtxn, &key)? { - Some(FacetGroupValue { bitmap, .. }) => bitmap, - None => { - error!( - "the facet value is missing from the facet database: {key:?}" - ); - continue; - } - }; - let count = search_candidates.intersection_len(&docids); - if count != 0 { - let value = self - .one_original_value_of(fid, value, docids.min().unwrap())? - .unwrap_or_else(|| query.to_string()); - results.push(FacetValueHit { value, count }); - length += 1; - } - if length >= MAX_NUMBER_OF_FACETS { + if self + .fetch_original_facets_using_normalized( + fid, + value, + query, + &search_candidates, + &mut results, + )? + .is_break() + { break; } } @@ -401,7 +384,6 @@ impl<'a> SearchForFacetValues<'a> { } None => { let mut results = vec![]; - let mut length = 0; let prefix = FacetGroupKey { field_id: fid, level: 0, left_bound: "" }; for result in index.facet_id_string_docids.prefix_iter(rtxn, &prefix)? { let (FacetGroupKey { left_bound, .. }, FacetGroupValue { bitmap, .. }) = @@ -412,9 +394,8 @@ impl<'a> SearchForFacetValues<'a> { .one_original_value_of(fid, left_bound, bitmap.min().unwrap())? .unwrap_or_else(|| left_bound.to_string()); results.push(FacetValueHit { value, count }); - length += 1; } - if length >= MAX_NUMBER_OF_FACETS { + if results.len() >= MAX_NUMBER_OF_FACETS { break; } } @@ -422,6 +403,50 @@ impl<'a> SearchForFacetValues<'a> { } } } + + fn fetch_original_facets_using_normalized( + &self, + fid: FieldId, + value: &str, + query: &str, + search_candidates: &RoaringBitmap, + results: &mut Vec, + ) -> Result> { + let index = self.search_query.index; + let rtxn = self.search_query.rtxn; + + let database = index.facet_id_normalized_string_strings; + let key = (fid, value); + let original_strings = match database.get(rtxn, &key)? { + Some(original_strings) => original_strings, + None => { + error!("the facet value is missing from the facet database: {key:?}"); + return Ok(ControlFlow::Continue(())); + } + }; + for original in original_strings { + let key = FacetGroupKey { field_id: fid, level: 0, left_bound: original.as_str() }; + let docids = match index.facet_id_string_docids.get(rtxn, &key)? { + Some(FacetGroupValue { bitmap, .. }) => bitmap, + None => { + error!("the facet value is missing from the facet database: {key:?}"); + return Ok(ControlFlow::Continue(())); + } + }; + let count = search_candidates.intersection_len(&docids); + if count != 0 { + let value = self + .one_original_value_of(fid, &original, docids.min().unwrap())? + .unwrap_or_else(|| query.to_string()); + results.push(FacetValueHit { value, count }); + } + if results.len() >= MAX_NUMBER_OF_FACETS { + return Ok(ControlFlow::Break(())); + } + } + + Ok(ControlFlow::Continue(())) + } } #[derive(Debug, Clone, serde::Serialize, PartialEq)] diff --git a/milli/src/search/new/distinct.rs b/milli/src/search/new/distinct.rs index fff96bd5d..e90ffe878 100644 --- a/milli/src/search/new/distinct.rs +++ b/milli/src/search/new/distinct.rs @@ -100,7 +100,7 @@ fn facet_number_values<'a>( } /// Return an iterator over each string value in the given field of the given document. -fn facet_string_values<'a>( +pub fn facet_string_values<'a>( docid: u32, field_id: u16, index: &Index, diff --git a/milli/src/search/new/geo_sort.rs b/milli/src/search/new/geo_sort.rs index dddb7f426..bd9546048 100644 --- a/milli/src/search/new/geo_sort.rs +++ b/milli/src/search/new/geo_sort.rs @@ -6,6 +6,7 @@ use heed::{RoPrefix, RoTxn}; use roaring::RoaringBitmap; use rstar::RTree; +use super::facet_string_values; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; use crate::heed_codec::facet::{FieldDocIdFacetCodec, OrderedF64Codec}; use crate::score_details::{self, ScoreDetails}; @@ -157,23 +158,7 @@ impl GeoSort { let mut documents = self .geo_candidates .iter() - .map(|id| -> Result<_> { - Ok(( - id, - [ - facet_number_values(id, lat, ctx.index, ctx.txn)? - .next() - .expect("A geo faceted document doesn't contain any lat")? - .0 - .2, - facet_number_values(id, lng, ctx.index, ctx.txn)? - .next() - .expect("A geo faceted document doesn't contain any lng")? - .0 - .2, - ], - )) - }) + .map(|id| -> Result<_> { Ok((id, geo_value(id, lat, lng, ctx.index, ctx.txn)?)) }) .collect::>>()?; // computing the distance between two points is expensive thus we cache the result documents @@ -185,6 +170,37 @@ impl GeoSort { } } +/// Extracts the lat and long values from a single document. +/// +/// If it is not able to find it in the facet number index it will extract it +/// from the facet string index and parse it as f64 (as the geo extraction behaves). +fn geo_value( + docid: u32, + field_lat: u16, + field_lng: u16, + index: &Index, + rtxn: &RoTxn, +) -> Result<[f64; 2]> { + let extract_geo = |geo_field: u16| -> Result { + match facet_number_values(docid, geo_field, index, rtxn)?.next() { + Some(Ok(((_, _, geo), ()))) => Ok(geo), + Some(Err(e)) => Err(e.into()), + None => match facet_string_values(docid, geo_field, index, rtxn)?.next() { + Some(Ok((_, geo))) => { + Ok(geo.parse::().expect("cannot parse geo field as f64")) + } + Some(Err(e)) => Err(e.into()), + None => panic!("A geo faceted document doesn't contain any lat or lng"), + }, + } + }; + + let lat = extract_geo(field_lat)?; + let lng = extract_geo(field_lng)?; + + Ok([lat, lng]) +} + impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for GeoSort { fn id(&self) -> String { "geo_sort".to_owned() diff --git a/milli/src/search/new/mod.rs b/milli/src/search/new/mod.rs index 8868d23fd..02f2d5642 100644 --- a/milli/src/search/new/mod.rs +++ b/milli/src/search/new/mod.rs @@ -28,7 +28,7 @@ use db_cache::DatabaseCache; use exact_attribute::ExactAttribute; use graph_based_ranking_rule::{Exactness, Fid, Position, Proximity, Typo}; use heed::RoTxn; -use hnsw::Searcher; +use instant_distance::Search; use interner::{DedupInterner, Interner}; pub use logger::visual::VisualSearchLogger; pub use logger::{DefaultSearchLogger, SearchLogger}; @@ -40,18 +40,18 @@ use ranking_rules::{ use resolve_query_graph::{compute_query_graph_docids, PhraseDocIdsCache}; use roaring::RoaringBitmap; use sort::Sort; -use space::Neighbor; +use self::distinct::facet_string_values; use self::geo_sort::GeoSort; pub use self::geo_sort::Strategy as GeoSortStrategy; use self::graph_based_ranking_rule::Words; use self::interner::Interned; +use crate::distance::NDotProductPoint; use crate::error::FieldIdMapMissingEntry; use crate::score_details::{ScoreDetails, ScoringStrategy}; use crate::search::new::distinct::apply_distinct_rule; use crate::{ - normalize_vector, AscDesc, DocumentId, Filter, Index, Member, Result, TermsMatchingStrategy, - UserError, BEU32, + AscDesc, DocumentId, Filter, Index, Member, Result, TermsMatchingStrategy, UserError, BEU32, }; /// A structure used throughout the execution of a search query. @@ -85,7 +85,12 @@ impl<'ctx> SearchContext<'ctx> { let searchable_names = self.index.searchable_fields(self.txn)?; let mut restricted_fids = Vec::new(); + let mut contains_wildcard = false; for field_name in searchable_attributes { + if field_name == "*" { + contains_wildcard = true; + continue; + } let searchable_contains_name = searchable_names.as_ref().map(|sn| sn.iter().any(|name| name == field_name)); let fid = match (fids_map.id(field_name), searchable_contains_name) { @@ -99,8 +104,10 @@ impl<'ctx> SearchContext<'ctx> { } .into()) } + // The field is not searchable, but the searchableAttributes are set to * => ignore field + (None, None) => continue, // The field is not searchable => User error - _otherwise => { + (_fid, Some(false)) => { let mut valid_fields: BTreeSet<_> = fids_map.names().map(String::from).collect(); @@ -132,7 +139,7 @@ impl<'ctx> SearchContext<'ctx> { restricted_fids.push(fid); } - self.restricted_fids = Some(restricted_fids); + self.restricted_fids = (!contains_wildcard).then_some(restricted_fids); Ok(()) } @@ -437,29 +444,31 @@ pub fn execute_search( check_sort_criteria(ctx, sort_criteria.as_ref())?; if let Some(vector) = vector { - let mut searcher = Searcher::new(); - let hnsw = ctx.index.vector_hnsw(ctx.txn)?.unwrap_or_default(); - let ef = hnsw.len().min(100); - let mut dest = vec![Neighbor { index: 0, distance: 0 }; ef]; - let vector = normalize_vector(vector.clone()); - let neighbors = hnsw.nearest(&vector, ef, &mut searcher, &mut dest[..]); + let mut search = Search::default(); + let docids = match ctx.index.vector_hnsw(ctx.txn)? { + Some(hnsw) => { + let vector = NDotProductPoint::new(vector.clone()); + let neighbors = hnsw.search(&vector, &mut search); - let mut docids = Vec::new(); - let mut uniq_docids = RoaringBitmap::new(); - for Neighbor { index, distance: _ } in neighbors.iter() { - let index = BEU32::new(*index as u32); - let docid = ctx.index.vector_id_docid.get(ctx.txn, &index)?.unwrap().get(); - if universe.contains(docid) && uniq_docids.insert(docid) { - docids.push(docid); - if docids.len() == (from + length) { - break; + let mut docids = Vec::new(); + let mut uniq_docids = RoaringBitmap::new(); + for instant_distance::Item { distance: _, pid, point: _ } in neighbors { + let index = BEU32::new(pid.into_inner()); + let docid = ctx.index.vector_id_docid.get(ctx.txn, &index)?.unwrap().get(); + if universe.contains(docid) && uniq_docids.insert(docid) { + docids.push(docid); + if docids.len() == (from + length) { + break; + } + } } - } - } - // return the nearest documents that are also part of the candidates - // along with a dummy list of scores that are useless in this context. - let docids: Vec<_> = docids.into_iter().skip(from).take(length).collect(); + // return the nearest documents that are also part of the candidates + // along with a dummy list of scores that are useless in this context. + docids.into_iter().skip(from).take(length).collect() + } + None => Vec::new(), + }; return Ok(PartialSearchResult { candidates: universe, diff --git a/milli/src/search/new/ranking_rule_graph/exactness/mod.rs b/milli/src/search/new/ranking_rule_graph/exactness/mod.rs index 0a84bf7cf..c5e58c635 100644 --- a/milli/src/search/new/ranking_rule_graph/exactness/mod.rs +++ b/milli/src/search/new/ranking_rule_graph/exactness/mod.rs @@ -1,7 +1,7 @@ use roaring::RoaringBitmap; use super::{ComputedCondition, RankingRuleGraphTrait}; -use crate::score_details::{Rank, ScoreDetails}; +use crate::score_details::{self, Rank, ScoreDetails}; use crate::search::new::interner::{DedupInterner, Interned}; use crate::search::new::query_term::{ExactTerm, LocatedQueryTermSubset}; use crate::search::new::resolve_query_graph::compute_query_term_subset_docids; @@ -87,6 +87,6 @@ impl RankingRuleGraphTrait for ExactnessGraph { } fn rank_to_score(rank: Rank) -> ScoreDetails { - ScoreDetails::Exactness(rank) + ScoreDetails::ExactWords(score_details::ExactWords::from_rank(rank)) } } diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap index ef95520bb..5463f0db5 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 9, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 9, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 8, + max_matching_words: 8, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 8, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 7, }, ), ], @@ -155,10 +155,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 7, }, ), ], @@ -175,10 +175,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -195,10 +195,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 5, }, ), ], @@ -215,10 +215,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -235,10 +235,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 4, }, ), ], @@ -255,10 +255,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -275,10 +275,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 3, }, ), ], @@ -295,10 +295,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -315,10 +315,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 2, }, ), ], @@ -335,10 +335,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -355,10 +355,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap index d48bf9933..bf3d9c403 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 7, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 7, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 7, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 1, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 0, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap index 991ff4cee..d1b24fcec 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 7, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap index 703f3cb7a..2e1e9fc56 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap index eb6141468..a7c0f4bc2 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap index 987e585a8..f2d94bf1a 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Typo( @@ -41,10 +41,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), Typo( @@ -67,10 +67,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), Typo( @@ -93,10 +93,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), Typo( @@ -119,10 +119,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 4, }, ), Typo( diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap index c5993a09e..88b937145 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 9, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 8, + max_matching_words: 8, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -155,10 +155,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -175,10 +175,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap index d920eb4a0..07585bc2d 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap index d0bc0fb46..4fb863a3d 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap index d0bc0fb46..4fb863a3d 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap index 21c6da724..5db8f8da8 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -41,10 +41,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -67,10 +67,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap index 7a33134cf..c99b50284 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -41,10 +41,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -67,10 +67,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -93,10 +93,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -119,10 +119,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -145,10 +145,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -171,10 +171,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -197,10 +197,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -223,10 +223,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap index 6670f3e4f..320e1b878 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap @@ -21,10 +21,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -47,10 +47,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), ], @@ -73,10 +73,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), ], @@ -99,10 +99,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 4, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap index ef95520bb..5463f0db5 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 9, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 9, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 8, + max_matching_words: 8, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 8, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 7, }, ), ], @@ -155,10 +155,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 7, }, ), ], @@ -175,10 +175,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -195,10 +195,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 5, }, ), ], @@ -215,10 +215,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -235,10 +235,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 4, }, ), ], @@ -255,10 +255,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -275,10 +275,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 3, }, ), ], @@ -295,10 +295,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -315,10 +315,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 2, }, ), ], @@ -335,10 +335,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -355,10 +355,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap index 1ca6a33a4..b12874c7b 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap @@ -37,10 +37,10 @@ expression: "format!(\"{document_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -78,10 +78,10 @@ expression: "format!(\"{document_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -119,10 +119,10 @@ expression: "format!(\"{document_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], diff --git a/milli/src/search/new/tests/stop_words.rs b/milli/src/search/new/tests/stop_words.rs index 4ad587240..2b06264f5 100644 --- a/milli/src/search/new/tests/stop_words.rs +++ b/milli/src/search/new/tests/stop_words.rs @@ -120,10 +120,10 @@ fn test_ignore_stop_words() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -173,10 +173,10 @@ fn test_ignore_stop_words() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -226,10 +226,10 @@ fn test_ignore_stop_words() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -278,10 +278,10 @@ fn test_ignore_stop_words() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -337,10 +337,10 @@ fn test_stop_words_in_phrase() { ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -378,10 +378,10 @@ fn test_stop_words_in_phrase() { ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -430,10 +430,10 @@ fn test_stop_words_in_phrase() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], diff --git a/milli/src/update/clear_documents.rs b/milli/src/update/clear_documents.rs index c021647ee..ab42fd854 100644 --- a/milli/src/update/clear_documents.rs +++ b/milli/src/update/clear_documents.rs @@ -36,6 +36,7 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> { script_language_docids, facet_id_f64_docids, facet_id_string_docids, + facet_id_normalized_string_strings, facet_id_string_fst, facet_id_exists_docids, facet_id_is_null_docids, @@ -94,6 +95,7 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> { word_prefix_fid_docids.clear(self.wtxn)?; script_language_docids.clear(self.wtxn)?; facet_id_f64_docids.clear(self.wtxn)?; + facet_id_normalized_string_strings.clear(self.wtxn)?; facet_id_string_fst.clear(self.wtxn)?; facet_id_exists_docids.clear(self.wtxn)?; facet_id_is_null_docids.clear(self.wtxn)?; diff --git a/milli/src/update/delete_documents.rs b/milli/src/update/delete_documents.rs index 48415adef..e23714530 100644 --- a/milli/src/update/delete_documents.rs +++ b/milli/src/update/delete_documents.rs @@ -4,10 +4,9 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use fst::IntoStreamer; use heed::types::{ByteSlice, DecodeIgnore, Str, UnalignedSlice}; use heed::{BytesDecode, BytesEncode, Database, RwIter}; -use hnsw::Searcher; +use instant_distance::PointId; use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; -use space::KnnPoints; use time::OffsetDateTime; use super::facet::delete::FacetsDelete; @@ -239,6 +238,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { word_prefix_fid_docids, facet_id_f64_docids: _, facet_id_string_docids: _, + facet_id_normalized_string_strings: _, facet_id_string_fst: _, field_id_docid_facet_f64s: _, field_id_docid_facet_strings: _, @@ -438,24 +438,24 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { // An ugly and slow way to remove the vectors from the HNSW // It basically reconstructs the HNSW from scratch without editing the current one. - let current_hnsw = self.index.vector_hnsw(self.wtxn)?.unwrap_or_default(); - if !current_hnsw.is_empty() { - let mut new_hnsw = Hnsw::default(); - let mut searcher = Searcher::new(); - let mut new_vector_id_docids = Vec::new(); - + if let Some(current_hnsw) = self.index.vector_hnsw(self.wtxn)? { + let mut points = Vec::new(); + let mut docids = Vec::new(); for result in vector_id_docid.iter(self.wtxn)? { let (vector_id, docid) = result?; if !self.to_delete_docids.contains(docid.get()) { - let vector = current_hnsw.get_point(vector_id.get() as usize).clone(); - let vector_id = new_hnsw.insert(vector, &mut searcher); - new_vector_id_docids.push((vector_id as u32, docid)); + let pid = PointId::from(vector_id.get()); + let vector = current_hnsw[pid].clone(); + points.push(vector); + docids.push(docid); } } + let (new_hnsw, pids) = Hnsw::builder().build_hnsw(points); + vector_id_docid.clear(self.wtxn)?; - for (vector_id, docid) in new_vector_id_docids { - vector_id_docid.put(self.wtxn, &BEU32::new(vector_id), &docid)?; + for (pid, docid) in pids.into_iter().zip(docids) { + vector_id_docid.put(self.wtxn, &BEU32::new(pid.into_inner()), &docid)?; } self.index.put_vector_hnsw(self.wtxn, &new_hnsw)?; } diff --git a/milli/src/update/facet/mod.rs b/milli/src/update/facet/mod.rs index 0e6fd494c..16fc1cd2f 100644 --- a/milli/src/update/facet/mod.rs +++ b/milli/src/update/facet/mod.rs @@ -76,9 +76,14 @@ pub const FACET_MAX_GROUP_SIZE: u8 = 8; pub const FACET_GROUP_SIZE: u8 = 4; pub const FACET_MIN_LEVEL_SIZE: u8 = 5; +use std::collections::BTreeSet; use std::fs::File; +use std::iter::FromIterator; -use heed::types::DecodeIgnore; +use charabia::normalizer::{Normalize, NormalizerOption}; +use grenad::{CompressionType, SortAlgorithm}; +use heed::types::{ByteSlice, DecodeIgnore, SerdeJson}; +use heed::BytesEncode; use log::debug; use time::OffsetDateTime; @@ -87,7 +92,9 @@ use super::FacetsUpdateBulk; use crate::facet::FacetType; use crate::heed_codec::facet::{FacetGroupKey, FacetGroupKeyCodec, FacetGroupValueCodec}; use crate::heed_codec::ByteSliceRefCodec; -use crate::{Index, Result, BEU16}; +use crate::update::index_documents::create_sorter; +use crate::update::merge_btreeset_string; +use crate::{BEU16StrCodec, Index, Result, BEU16}; pub mod bulk; pub mod delete; @@ -159,26 +166,69 @@ impl<'i> FacetsUpdate<'i> { incremental_update.execute(wtxn)?; } + // We clear the list of normalized-for-search facets + // and the previous FSTs to compute everything from scratch + self.index.facet_id_normalized_string_strings.clear(wtxn)?; + self.index.facet_id_string_fst.clear(wtxn)?; + + // As we can't use the same write transaction to read and write in two different databases + // we must create a temporary sorter that we will write into LMDB afterward. + // As multiple unnormalized facet values can become the same normalized facet value + // we must merge them together. + let mut sorter = create_sorter( + SortAlgorithm::Unstable, + merge_btreeset_string, + CompressionType::None, + None, + None, + None, + ); + + // We iterate on the list of original, semi-normalized, facet values + // and normalize them for search, inserting them in LMDB in any given order. + let options = NormalizerOption { lossy: true, ..Default::default() }; + let database = self.index.facet_id_string_docids.remap_data_type::(); + for result in database.iter(wtxn)? { + let (facet_group_key, ()) = result?; + if let FacetGroupKey { field_id, level: 0, left_bound } = facet_group_key { + let normalized_facet = left_bound.normalize(&options); + let set = BTreeSet::from_iter(std::iter::once(left_bound)); + let key = (field_id, normalized_facet.as_ref()); + let key = BEU16StrCodec::bytes_encode(&key).ok_or(heed::Error::Encoding)?; + let val = SerdeJson::bytes_encode(&set).ok_or(heed::Error::Encoding)?; + sorter.insert(key, val)?; + } + } + + // In this loop we don't need to take care of merging bitmaps + // as the grenad sorter already merged them for us. + let mut merger_iter = sorter.into_stream_merger_iter()?; + while let Some((key_bytes, btreeset_bytes)) = merger_iter.next()? { + self.index + .facet_id_normalized_string_strings + .remap_types::() + .put(wtxn, key_bytes, btreeset_bytes)?; + } + // We compute one FST by string facet let mut text_fsts = vec![]; let mut current_fst: Option<(u16, fst::SetBuilder>)> = None; - let database = self.index.facet_id_string_docids.remap_data_type::(); + let database = + self.index.facet_id_normalized_string_strings.remap_data_type::(); for result in database.iter(wtxn)? { - let (facet_group_key, _) = result?; - if let FacetGroupKey { field_id, level: 0, left_bound } = facet_group_key { - current_fst = match current_fst.take() { - Some((fid, fst_builder)) if fid != field_id => { - let fst = fst_builder.into_set(); - text_fsts.push((fid, fst)); - Some((field_id, fst::SetBuilder::memory())) - } - Some((field_id, fst_builder)) => Some((field_id, fst_builder)), - None => Some((field_id, fst::SetBuilder::memory())), - }; - - if let Some((_, fst_builder)) = current_fst.as_mut() { - fst_builder.insert(left_bound)?; + let ((field_id, normalized_facet), _) = result?; + current_fst = match current_fst.take() { + Some((fid, fst_builder)) if fid != field_id => { + let fst = fst_builder.into_set(); + text_fsts.push((fid, fst)); + Some((field_id, fst::SetBuilder::memory())) } + Some((field_id, fst_builder)) => Some((field_id, fst_builder)), + None => Some((field_id, fst::SetBuilder::memory())), + }; + + if let Some((_, fst_builder)) = current_fst.as_mut() { + fst_builder.insert(normalized_facet)?; } } @@ -187,9 +237,6 @@ impl<'i> FacetsUpdate<'i> { text_fsts.push((field_id, fst)); } - // We remove all of the previous FSTs that were in this database - self.index.facet_id_string_fst.clear(wtxn)?; - // We write those FSTs in LMDB now for (field_id, fst) in text_fsts { self.index.facet_id_string_fst.put(wtxn, &BEU16::new(field_id), &fst)?; diff --git a/milli/src/update/index_documents/helpers/merge_functions.rs b/milli/src/update/index_documents/helpers/merge_functions.rs index 64bee95df..5d111067a 100644 --- a/milli/src/update/index_documents/helpers/merge_functions.rs +++ b/milli/src/update/index_documents/helpers/merge_functions.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::collections::BTreeSet; use std::io; use std::result::Result as StdResult; @@ -44,6 +45,27 @@ pub fn merge_roaring_bitmaps<'a>(_key: &[u8], values: &[Cow<'a, [u8]>]) -> Resul } } +pub fn merge_btreeset_string<'a>(_key: &[u8], values: &[Cow<'a, [u8]>]) -> Result> { + if values.len() == 1 { + Ok(values[0].clone()) + } else { + // TODO improve the perf by using a `#[borrow] Cow`. + let strings: BTreeSet = values + .iter() + .map(AsRef::as_ref) + .map(serde_json::from_slice::>) + .map(StdResult::unwrap) + .reduce(|mut current, new| { + for x in new { + current.insert(x); + } + current + }) + .unwrap(); + Ok(Cow::Owned(serde_json::to_vec(&strings).unwrap())) + } +} + pub fn keep_first<'a>(_key: &[u8], values: &[Cow<'a, [u8]>]) -> Result> { Ok(values[0].clone()) } diff --git a/milli/src/update/index_documents/helpers/mod.rs b/milli/src/update/index_documents/helpers/mod.rs index 95e497af4..d59a3bc08 100644 --- a/milli/src/update/index_documents/helpers/mod.rs +++ b/milli/src/update/index_documents/helpers/mod.rs @@ -13,9 +13,9 @@ pub use grenad_helpers::{ GrenadParameters, MergeableReader, }; pub use merge_functions::{ - concat_u32s_array, keep_first, keep_latest_obkv, merge_cbo_roaring_bitmaps, - merge_obkvs_and_operations, merge_roaring_bitmaps, merge_two_obkvs, serialize_roaring_bitmap, - MergeFn, + concat_u32s_array, keep_first, keep_latest_obkv, merge_btreeset_string, + merge_cbo_roaring_bitmaps, merge_obkvs_and_operations, merge_roaring_bitmaps, merge_two_obkvs, + serialize_roaring_bitmap, MergeFn, }; use crate::MAX_WORD_LENGTH; diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 9a657674e..5426e26db 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -26,7 +26,7 @@ pub use self::enrich::{ }; pub use self::helpers::{ as_cloneable_grenad, create_sorter, create_writer, fst_stream_into_hashset, - fst_stream_into_vec, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, + fst_stream_into_vec, merge_btreeset_string, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, sorter_into_lmdb_database, valid_lmdb_key, writer_into_reader, ClonableMmap, MergeFn, }; use self::helpers::{grenad_obkv_into_chunks, GrenadParameters}; diff --git a/milli/src/update/index_documents/typed_chunk.rs b/milli/src/update/index_documents/typed_chunk.rs index 96dea0480..788aaf93d 100644 --- a/milli/src/update/index_documents/typed_chunk.rs +++ b/milli/src/update/index_documents/typed_chunk.rs @@ -9,22 +9,19 @@ use charabia::{Language, Script}; use grenad::MergerBuilder; use heed::types::ByteSlice; use heed::RwTxn; -use hnsw::Searcher; use roaring::RoaringBitmap; -use space::KnnPoints; use super::helpers::{ self, merge_ignore_values, serialize_roaring_bitmap, valid_lmdb_key, CursorClonableMmap, }; use super::{ClonableMmap, MergeFn}; +use crate::distance::NDotProductPoint; use crate::error::UserError; use crate::facet::FacetType; +use crate::index::Hnsw; use crate::update::facet::FacetsUpdate; use crate::update::index_documents::helpers::{as_cloneable_grenad, try_split_array_at}; -use crate::{ - lat_lng_to_xyz, normalize_vector, CboRoaringBitmapCodec, DocumentId, GeoPoint, Index, Result, - BEU32, -}; +use crate::{lat_lng_to_xyz, CboRoaringBitmapCodec, DocumentId, GeoPoint, Index, Result, BEU32}; pub(crate) enum TypedChunk { FieldIdDocidFacetStrings(grenad::Reader), @@ -292,17 +289,20 @@ pub(crate) fn write_typed_chunk_into_index( index.put_geo_faceted_documents_ids(wtxn, &geo_faceted_docids)?; } TypedChunk::VectorPoints(vector_points) => { - let mut hnsw = index.vector_hnsw(wtxn)?.unwrap_or_default(); - let mut searcher = Searcher::new(); - - let mut expected_dimensions = match index.vector_id_docid.iter(wtxn)?.next() { - Some(result) => { - let (vector_id, _) = result?; - Some(hnsw.get_point(vector_id.get() as usize).len()) - } - None => None, + let (pids, mut points): (Vec<_>, Vec<_>) = match index.vector_hnsw(wtxn)? { + Some(hnsw) => hnsw.iter().map(|(pid, point)| (pid, point.clone())).unzip(), + None => Default::default(), }; + // Convert the PointIds into DocumentIds + let mut docids = Vec::new(); + for pid in pids { + let docid = + index.vector_id_docid.get(wtxn, &BEU32::new(pid.into_inner()))?.unwrap(); + docids.push(docid.get()); + } + + let mut expected_dimensions = points.get(0).map(|p| p.len()); let mut cursor = vector_points.into_cursor()?; while let Some((key, value)) = cursor.move_on_next()? { // convert the key back to a u32 (4 bytes) @@ -318,12 +318,26 @@ pub(crate) fn write_typed_chunk_into_index( return Err(UserError::InvalidVectorDimensions { expected, found })?; } - let vector = normalize_vector(vector); - let vector_id = hnsw.insert(vector, &mut searcher) as u32; - index.vector_id_docid.put(wtxn, &BEU32::new(vector_id), &BEU32::new(docid))?; + points.push(NDotProductPoint::new(vector)); + docids.push(docid); } - log::debug!("There are {} entries in the HNSW so far", hnsw.len()); - index.put_vector_hnsw(wtxn, &hnsw)?; + + assert_eq!(docids.len(), points.len()); + + let hnsw_length = points.len(); + let (new_hnsw, pids) = Hnsw::builder().build_hnsw(points); + + index.vector_id_docid.clear(wtxn)?; + for (docid, pid) in docids.into_iter().zip(pids) { + index.vector_id_docid.put( + wtxn, + &BEU32::new(pid.into_inner()), + &BEU32::new(docid), + )?; + } + + log::debug!("There are {} entries in the HNSW so far", hnsw_length); + index.put_vector_hnsw(wtxn, &new_hnsw)?; } TypedChunk::ScriptLanguageDocids(hash_pair) => { let mut buffer = Vec::new(); diff --git a/milli/src/update/mod.rs b/milli/src/update/mod.rs index 32584825b..9982957e5 100644 --- a/milli/src/update/mod.rs +++ b/milli/src/update/mod.rs @@ -4,8 +4,9 @@ pub use self::delete_documents::{DeleteDocuments, DeletionStrategy, DocumentDele pub use self::facet::bulk::FacetsUpdateBulk; pub use self::facet::incremental::FacetsUpdateIncrementalInner; pub use self::index_documents::{ - merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, DocumentAdditionResult, DocumentId, - IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, MergeFn, + merge_btreeset_string, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, + DocumentAdditionResult, DocumentId, IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, + MergeFn, }; pub use self::indexer_config::IndexerConfig; pub use self::prefix_word_pairs::{ diff --git a/milli/src/update/settings.rs b/milli/src/update/settings.rs index 2ae5077f8..7e52c04c1 100644 --- a/milli/src/update/settings.rs +++ b/milli/src/update/settings.rs @@ -466,13 +466,14 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { let current = self.index.stop_words(self.wtxn)?; // Apply an unlossy normalization on stop_words - let stop_words = stop_words + let stop_words: BTreeSet = stop_words .iter() - .map(|w| w.as_str().normalize(&Default::default()).into_owned()); + .map(|w| w.as_str().normalize(&Default::default()).into_owned()) + .collect(); // since we can't compare a BTreeSet with an FST we are going to convert the // BTreeSet to an FST and then compare bytes per bytes the two FSTs. - let fst = fst::Set::from_iter(stop_words)?; + let fst = fst::Set::from_iter(stop_words.into_iter())?; // Does the new FST differ from the previous one? if current