diff --git a/.github/workflows/benchmarks-manual.yml b/.github/workflows/benchmarks-manual.yml index da33bf803..14b77c83d 100644 --- a/.github/workflows/benchmarks-manual.yml +++ b/.github/workflows/benchmarks-manual.yml @@ -43,7 +43,7 @@ jobs: # Run benchmarks - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} run: | - cd benchmarks + cd crates/benchmarks cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }} # Generate critcmp files diff --git a/.github/workflows/benchmarks-pr.yml b/.github/workflows/benchmarks-pr.yml index f9d609d6e..a083baa3c 100644 --- a/.github/workflows/benchmarks-pr.yml +++ b/.github/workflows/benchmarks-pr.yml @@ -88,7 +88,7 @@ jobs: # Run benchmarks - name: Run benchmarks - Dataset ${{ steps.command.outputs.command-arguments }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} run: | - cd benchmarks + cd crates/benchmarks cargo bench --bench ${{ steps.command.outputs.command-arguments }} -- --save-baseline ${{ steps.file.outputs.basename }} # Generate critcmp files diff --git a/.github/workflows/benchmarks-push-indexing.yml b/.github/workflows/benchmarks-push-indexing.yml index 1fdd5fd67..4495b4b9d 100644 --- a/.github/workflows/benchmarks-push-indexing.yml +++ b/.github/workflows/benchmarks-push-indexing.yml @@ -41,7 +41,7 @@ jobs: # Run benchmarks - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} run: | - cd benchmarks + cd crates/benchmarks cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }} # Generate critcmp files diff --git a/.github/workflows/benchmarks-push-search-geo.yml b/.github/workflows/benchmarks-push-search-geo.yml index 82881b41b..22218cd6e 100644 --- a/.github/workflows/benchmarks-push-search-geo.yml +++ b/.github/workflows/benchmarks-push-search-geo.yml @@ -40,7 +40,7 @@ jobs: # Run benchmarks - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} run: | - cd benchmarks + cd crates/benchmarks cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }} # Generate critcmp files diff --git a/.github/workflows/benchmarks-push-search-songs.yml b/.github/workflows/benchmarks-push-search-songs.yml index b6169ddf7..e9744a434 100644 --- a/.github/workflows/benchmarks-push-search-songs.yml +++ b/.github/workflows/benchmarks-push-search-songs.yml @@ -40,7 +40,7 @@ jobs: # Run benchmarks - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} run: | - cd benchmarks + cd crates/benchmarks cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }} # Generate critcmp files diff --git a/.github/workflows/benchmarks-push-search-wiki.yml b/.github/workflows/benchmarks-push-search-wiki.yml index dd3146a14..bc9e1bcd0 100644 --- a/.github/workflows/benchmarks-push-search-wiki.yml +++ b/.github/workflows/benchmarks-push-search-wiki.yml @@ -40,7 +40,7 @@ jobs: # Run benchmarks - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} run: | - cd benchmarks + cd crates/benchmarks cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }} # Generate critcmp files diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 016a9d282..c53946fea 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -65,9 +65,9 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-12, windows-2022] + os: [macos-13, windows-2022] include: - - os: macos-12 + - os: macos-13 artifact_name: meilisearch asset_name: meilisearch-macos-amd64 - os: windows-2022 @@ -90,7 +90,7 @@ jobs: publish-macos-apple-silicon: name: Publish binary for macOS silicon - runs-on: macos-12 + runs-on: macos-13 needs: check-version strategy: matrix: diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 7dbd7d866..e142b15b6 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -33,7 +33,7 @@ jobs: - name: Setup test with Rust stable uses: dtolnay/rust-toolchain@1.79 - name: Cache dependencies - uses: Swatinem/rust-cache@v2.7.1 + uses: Swatinem/rust-cache@v2.7.5 - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: @@ -51,11 +51,11 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-12, windows-2022] + os: [macos-13, windows-2022] steps: - uses: actions/checkout@v3 - name: Cache dependencies - uses: Swatinem/rust-cache@v2.7.1 + uses: Swatinem/rust-cache@v2.7.5 - uses: dtolnay/rust-toolchain@1.79 - name: Run cargo check without any default features uses: actions-rs/cargo@v1 @@ -127,7 +127,7 @@ jobs: apt-get install build-essential -y - uses: dtolnay/rust-toolchain@1.79 - name: Cache dependencies - uses: Swatinem/rust-cache@v2.7.1 + uses: Swatinem/rust-cache@v2.7.5 - name: Run tests in debug uses: actions-rs/cargo@v1 with: @@ -144,7 +144,7 @@ jobs: profile: minimal components: clippy - name: Cache dependencies - uses: Swatinem/rust-cache@v2.7.1 + uses: Swatinem/rust-cache@v2.7.5 - name: Run cargo clippy uses: actions-rs/cargo@v1 with: @@ -163,11 +163,11 @@ jobs: override: true components: rustfmt - name: Cache dependencies - uses: Swatinem/rust-cache@v2.7.1 + uses: Swatinem/rust-cache@v2.7.5 - name: Run cargo fmt # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file. # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate # we are going to create an empty file where rustfmt expects it. run: | - echo -ne "\n" > benchmarks/benches/datasets_paths.rs + echo -ne "\n" > crates/benchmarks/benches/datasets_paths.rs cargo fmt --all -- --check diff --git a/.gitignore b/.gitignore index e00f45c1e..0d6750008 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ **/*.json_lines **/*.rs.bk /*.mdb -/query-history.txt /data.ms /snapshots /dumps @@ -19,4 +18,4 @@ *.snap.new # Fuzzcheck data for the facet indexing fuzz test -milli/fuzz/update::facet::incremental::fuzz::fuzz/ +crates/milli/fuzz/update::facet::incremental::fuzz::fuzz/ diff --git a/Cargo.lock b/Cargo.lock index b49000574..c09c28d25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,11 +4,11 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ - "bitflags 2.6.0", + "bitflags 1.3.2", "bytes", "futures-core", "futures-sink", @@ -36,9 +36,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.9.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48f96fc3003717aeb9856ca3d02a8c7de502667ad76eeacd830b48d2e91fac4" +checksum = "3ae682f693a9cd7b058f2b0b5d9a6d7728a8555779bedbbc35dd88528611d020" dependencies = [ "actix-codec", "actix-rt", @@ -56,7 +56,7 @@ dependencies = [ "flate2", "futures-core", "h2 0.3.26", - "http 0.2.12", + "http 0.2.11", "httparse", "httpdate", "itoa", @@ -80,7 +80,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -91,7 +91,7 @@ checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" dependencies = [ "bytestring", "cfg-if", - "http 0.2.12", + "http 0.2.11", "regex-lite", "serde", "tracing", @@ -110,9 +110,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.5.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca2549781d8dd6d75c40cf6b6051260a2cc2f3c62343d761a969a0640646894" +checksum = "3e8613a75dd50cc45f473cee3c34d59ed677c0f7b44480ce3b8247d7dc519327" dependencies = [ "actix-rt", "actix-service", @@ -120,7 +120,8 @@ dependencies = [ "futures-core", "futures-util", "mio", - "socket2", + "num_cpus", + "socket2 0.4.9", "tokio", "tracing", ] @@ -167,9 +168,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.9.0" +version = "4.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9180d76e5cc7ccbc4d60a506f2c727730b154010262df5b910eb17dbe4b8cb38" +checksum = "1988c02af8d2b718c05bc4aeb6a66395b7cdf32858c2c71131e5637a8c05a9ff" dependencies = [ "actix-codec", "actix-http", @@ -190,7 +191,6 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "impl-more", "itoa", "language-tags", "log", @@ -202,7 +202,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", + "socket2 0.5.5", "time", "url", ] @@ -216,23 +216,23 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] name = "addr2line" -version = "0.24.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" dependencies = [ "gimli", ] [[package]] -name = "adler2" -version = "2.0.0" +name = "adler" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aes" @@ -308,58 +308,57 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.17" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] name = "anyhow" -version = "1.0.91" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" dependencies = [ "backtrace", ] @@ -381,14 +380,34 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arroy" -version = "0.4.0" -source = "git+https://github.com/meilisearch/arroy/?rev=2386594dfb009ce08821a925ccc89fb8e30bf73d#2386594dfb009ce08821a925ccc89fb8e30bf73d" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc5f272f38fa063bbff0a7ab5219404e221493de005e2b4078c62d626ef567e" +dependencies = [ + "bytemuck", + "byteorder", + "heed", + "log", + "memmap2", + "nohash", + "ordered-float", + "rand", + "rayon", + "roaring", + "tempfile", + "thiserror", +] + +[[package]] +name = "arroy" +version = "0.5.0" +source = "git+https://github.com/meilisearch/arroy/?tag=DO-NOT-DELETE-upgrade-v04-to-v05#053807bf38dc079f25b003f19fc30fbf3613f6e7" dependencies = [ "bytemuck", "byteorder", @@ -416,13 +435,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -433,23 +452,23 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.4.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "backtrace" -version = "0.3.74" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" dependencies = [ "addr2line", + "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", - "windows-targets 0.52.6", ] [[package]] @@ -515,20 +534,22 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.70.1" +version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ "bitflags 2.6.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.12.1", + "lazy_static", + "lazycell", "proc-macro2", "quote", "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -622,7 +643,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "syn_derive", ] @@ -649,9 +670,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.10.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", "regex-automata", @@ -712,9 +733,9 @@ dependencies = [ [[package]] name = "bytecount" -version = "0.6.8" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" +checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" [[package]] name = "bytemuck" @@ -727,13 +748,13 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.8.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" +checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -744,15 +765,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "bytestring" -version = "1.3.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" +checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" dependencies = [ "bytes", ] @@ -780,9 +801,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.9" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" dependencies = [ "serde", ] @@ -797,7 +818,7 @@ dependencies = [ "candle-kernels", "cudarc", "gemm", - "half", + "half 2.4.0", "memmap2", "num-traits", "num_cpus", @@ -826,7 +847,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b006b30f66a0d94fc9cef0ac4de6ce510565f35ae2c6c35ce5d4aacfb0fc8eeb" dependencies = [ "candle-core", - "half", + "half 2.4.0", "num-traits", "rayon", "safetensors", @@ -855,9 +876,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.8" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" dependencies = [ "serde", ] @@ -878,9 +899,9 @@ dependencies = [ [[package]] name = "cargo_toml" -version = "0.20.5" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88da5a13c620b4ca0078845707ea9c3faf11edbc3ffd8497d11d686211cd1ac0" +checksum = "4895c018bb228aa6b3ba1a0285543fcb4b704734c3fb1f72afaa75aa769500c1" dependencies = [ "serde", "toml", @@ -894,13 +915,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.1.31" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" dependencies = [ "jobserver", "libc", - "shlex", + "once_cell", ] [[package]] @@ -966,9 +987,9 @@ dependencies = [ [[package]] name = "ciborium" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" dependencies = [ "ciborium-io", "ciborium-ll", @@ -977,18 +998,18 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" [[package]] name = "ciborium-ll" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" dependencies = [ "ciborium-io", - "half", + "half 1.8.2", ] [[package]] @@ -1003,9 +1024,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.8.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", @@ -1014,9 +1035,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" dependencies = [ "clap_builder", "clap_derive", @@ -1024,9 +1045,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" dependencies = [ "anstream", "anstyle", @@ -1036,21 +1057,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "color-spantrace" @@ -1066,9 +1087,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "concat-arrays" @@ -1083,15 +1104,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.8" +version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.52.0", + "windows-sys 0.45.0", ] [[package]] @@ -1116,9 +1137,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "convert_case" @@ -1148,15 +1169,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.7" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -1316,11 +1337,11 @@ dependencies = [ [[package]] name = "cudarc" -version = "0.11.9" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bd4d1eee570c3b2ac64ed114125517dd1e541d88dd28fc259f1de4dba8d60" +checksum = "56ee2a3fbbd981e1c7ea73cc2af136e754eb22d17436de37155227ee4dbe0cf4" dependencies = [ - "half", + "half 2.4.0", "libloading", ] @@ -1336,12 +1357,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.10" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" dependencies = [ - "darling_core 0.20.10", - "darling_macro 0.20.10", + "darling_core 0.20.9", + "darling_macro 0.20.9", ] [[package]] @@ -1360,16 +1381,16 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.10" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1385,13 +1406,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.10" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ - "darling_core 0.20.10", + "darling_core 0.20.9", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1423,9 +1444,9 @@ dependencies = [ [[package]] name = "deflate64" -version = "0.1.9" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" +checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" [[package]] name = "deranged" @@ -1445,7 +1466,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1459,11 +1480,11 @@ dependencies = [ [[package]] name = "derive_builder" -version = "0.20.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" dependencies = [ - "derive_builder_macro 0.20.2", + "derive_builder_macro 0.20.0", ] [[package]] @@ -1480,14 +1501,14 @@ dependencies = [ [[package]] name = "derive_builder_core" -version = "0.20.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ - "darling 0.20.10", + "darling 0.20.9", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1502,25 +1523,25 @@ dependencies = [ [[package]] name = "derive_builder_macro" -version = "0.20.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ - "derive_builder_core 0.20.2", - "syn 2.0.85", + "derive_builder_core 0.20.0", + "syn 2.0.87", ] [[package]] name = "derive_more" -version = "0.99.18" +version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", - "syn 2.0.85", + "syn 1.0.109", ] [[package]] @@ -1549,7 +1570,7 @@ dependencies = [ "convert_case 0.6.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1613,7 +1634,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1746,9 +1767,9 @@ checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] @@ -1764,14 +1785,14 @@ dependencies = [ [[package]] name = "enum-as-inner" -version = "0.6.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1791,7 +1812,7 @@ checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1802,9 +1823,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1839,9 +1860,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "file-store" @@ -1855,14 +1876,14 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.25" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" dependencies = [ "cfg-if", "libc", - "libredox", - "windows-sys 0.59.0", + "redox_syscall 0.3.5", + "windows-sys 0.48.0", ] [[package]] @@ -1877,9 +1898,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.34" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -1928,9 +1949,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1943,9 +1964,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1953,15 +1974,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1970,38 +1991,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -2109,7 +2130,7 @@ checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" dependencies = [ "bytemuck", "dyn-stack", - "half", + "half 2.4.0", "num-complex", "num-traits", "once_cell", @@ -2130,7 +2151,7 @@ dependencies = [ "dyn-stack", "gemm-common", "gemm-f32", - "half", + "half 2.4.0", "num-complex", "num-traits", "paste", @@ -2187,9 +2208,9 @@ checksum = "36d244a08113319b5ebcabad2b8b7925732d15eec46d7e7ac3c11734f3b7a6ad" [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "js-sys", @@ -2199,10 +2220,22 @@ dependencies = [ ] [[package]] -name = "gimli" -version = "0.31.1" +name = "getset" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" [[package]] name = "git2" @@ -2246,7 +2279,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.12", + "http 0.2.11", "indexmap", "slab", "tokio", @@ -2256,9 +2289,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ "atomic-waker", "bytes", @@ -2275,9 +2308,15 @@ dependencies = [ [[package]] name = "half" -version = "2.4.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "half" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" dependencies = [ "bytemuck", "cfg-if", @@ -2307,9 +2346,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ "ahash 0.8.11", "allocator-api2", @@ -2317,9 +2356,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" dependencies = [ "allocator-api2", "equivalent", @@ -2337,6 +2376,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -2345,9 +2390,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heed" -version = "0.20.5" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d4f449bab7320c56003d37732a917e18798e2f1709d80263face2b4f9436ddb" +checksum = "2bc30da4a93ff8cb98e535d595d6de42731d4719d707bc1c86f579158751a24e" dependencies = [ "bitflags 2.6.0", "byteorder", @@ -2386,12 +2431,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" -[[package]] -name = "hermit-abi" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" - [[package]] name = "hex" version = "0.4.3" @@ -2425,9 +2464,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.12" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -2447,9 +2486,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", "http 1.1.0", @@ -2470,26 +2509,26 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "1.5.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.6", + "h2 0.4.5", "http 1.1.0", "http-body", "httparse", @@ -2503,9 +2542,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", @@ -2521,9 +2560,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" dependencies = [ "bytes", "futures-channel", @@ -2532,8 +2571,9 @@ dependencies = [ "http-body", "hyper", "pin-project-lite", - "socket2", + "socket2 0.5.5", "tokio", + "tower", "tower-service", "tracing", ] @@ -2556,22 +2596,22 @@ dependencies = [ [[package]] name = "impl-more" -version = "0.1.8" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" +checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d" [[package]] name = "index-scheduler" version = "1.11.0" dependencies = [ "anyhow", - "arroy", + "arroy 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "big_s", "bincode", "bumpalo", "crossbeam", "csv", - "derive_builder 0.20.2", + "derive_builder 0.20.0", "dump", "enum-iterator", "file-store", @@ -2598,20 +2638,20 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.14.3", "serde", ] [[package]] name = "indicatif" -version = "0.17.8" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" dependencies = [ "console", "instant", @@ -2646,18 +2686,18 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.13" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", ] [[package]] name = "ipnet" -version = "2.10.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "irg-kvariants" @@ -2671,21 +2711,15 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.13" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.4.0", + "hermit-abi", "libc", "windows-sys 0.52.0", ] -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - [[package]] name = "itertools" version = "0.10.5" @@ -2735,7 +2769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e2b0210dc78b49337af9e49d7ae41a39dceac6e5985613f1cf7763e2f76a25" dependencies = [ "cedarwood", - "derive_builder 0.20.2", + "derive_builder 0.20.0", "fxhash", "lazy_static", "phf", @@ -2745,18 +2779,18 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -2795,9 +2829,9 @@ dependencies = [ [[package]] name = "kstring" -version = "2.0.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +checksum = "ec3066350882a1cd6d950d055997f379ac37fd39f81cd4d8ed186032eb3c5747" dependencies = [ "serde", "static_assertions", @@ -2815,6 +2849,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "levenshtein_automata" version = "0.2.1" @@ -2826,9 +2866,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.161" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libgit2-sys" @@ -2844,19 +2884,19 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.4", ] [[package]] name = "libm" -version = "0.2.11" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" @@ -2870,31 +2910,20 @@ dependencies = [ [[package]] name = "libproc" -version = "0.14.10" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb" +checksum = "ae9ea4b75e1a81675429dafe43441df1caea70081e82246a8cccf514884a88bb" dependencies = [ "bindgen", "errno", "libc", ] -[[package]] -name = "libredox" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags 2.6.0", - "libc", - "redox_syscall", -] - [[package]] name = "libz-sys" -version = "1.1.20" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" +checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" dependencies = [ "cc", "libc", @@ -3058,7 +3087,7 @@ dependencies = [ "bincode", "byteorder", "csv", - "derive_builder 0.20.2", + "derive_builder 0.20.0", "encoding", "encoding_rs", "encoding_rs_io", @@ -3223,15 +3252,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "liquid" -version = "0.26.9" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cdcc72b82748f47c2933c172313f5a9aea5b2c4eb3fa4c66b4ea55bb60bb4b1" +checksum = "10929f201279ba14da3297b957dcda1e0bf7a6f3bb5115688be684aa8864e9cc" dependencies = [ "doc-comment", "liquid-core", @@ -3242,12 +3271,12 @@ dependencies = [ [[package]] name = "liquid-core" -version = "0.26.9" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2752e978ffc53670f3f2e8b3ef09f348d6f7b5474a3be3f8a5befe5382e4effb" +checksum = "3aef4b2160791f456eb880c990a97746f693746f92302ef5f1d06111cf14b768" dependencies = [ "anymap2", - "itertools 0.13.0", + "itertools 0.12.1", "kstring", "liquid-derive", "num-traits", @@ -3260,22 +3289,22 @@ dependencies = [ [[package]] name = "liquid-derive" -version = "0.26.8" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b51f1d220e3fa869e24cfd75915efe3164bd09bb11b3165db3f37f57bf673e3" +checksum = "915f6d0a2963a27cd5205c1902f32ddfe3bc035816afd268cf88c0fc0f8d287e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] name = "liquid-lib" -version = "0.26.9" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b1a298d3d2287ee5b1e43840d885b8fdfc37d3f4e90d82aacfd04d021618da" +checksum = "73f48fc446873f74d869582f5c4b8cbf3248c93395e410a67af5809b3731e44a" dependencies = [ - "itertools 0.13.0", + "itertools 0.12.1", "liquid-core", "once_cell", "percent-encoding", @@ -3286,9 +3315,9 @@ dependencies = [ [[package]] name = "lmdb-master-sys" -version = "0.2.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "472c3760e2a8d0f61f322fb36788021bb36d573c502b50fa3e2bcaac3ec326c9" +checksum = "57640c190703d5ccf4a86aff4aeb749b2d287a8cb1723c76b51f39d77ab53b24" dependencies = [ "cc", "doxygen-rs", @@ -3297,26 +3326,27 @@ dependencies = [ [[package]] name = "local-channel" -version = "0.1.5" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" +checksum = "7f303ec0e94c6c54447f84f3b0ef7af769858a9c4ef56ef2a986d3dcd4c3fc9c" dependencies = [ "futures-core", "futures-sink", + "futures-util", "local-waker", ] [[package]] name = "local-waker" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" +checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -3330,9 +3360,9 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" [[package]] name = "log" -version = "0.4.22" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lzma-rs" @@ -3369,7 +3399,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3435,6 +3465,7 @@ dependencies = [ "meilisearch-types", "mimalloc", "mime", + "mopa-maintained", "num_cpus", "obkv", "once_cell", @@ -3479,7 +3510,7 @@ dependencies = [ "uuid", "wiremock", "yaup", - "zip 2.2.0", + "zip 2.1.3", ] [[package]] @@ -3536,6 +3567,7 @@ name = "meilitool" version = "1.11.0" dependencies = [ "anyhow", + "arroy 0.5.0 (git+https://github.com/meilisearch/arroy/?tag=DO-NOT-DELETE-upgrade-v04-to-v05)", "clap", "dump", "file-store", @@ -3554,9 +3586,9 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" -version = "0.9.5" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" dependencies = [ "libc", "stable_deref_trait", @@ -3567,7 +3599,7 @@ name = "milli" version = "1.11.0" dependencies = [ "allocator-api2", - "arroy", + "arroy 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "big_s", "bimap", "bincode", @@ -3590,7 +3622,7 @@ dependencies = [ "fxhash", "geoutils", "grenad", - "hashbrown 0.15.0", + "hashbrown 0.15.1", "heed", "hf-hub", "indexmap", @@ -3651,9 +3683,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" -version = "2.0.5" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" dependencies = [ "mime", "unicase", @@ -3667,31 +3699,30 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ - "adler2", + "adler", ] [[package]] name = "mio" -version = "1.0.2" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ - "hermit-abi 0.3.9", "libc", "log", "wasi", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "monostate" -version = "0.1.13" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" +checksum = "15f370ae88093ec6b11a710dec51321a61d420fafd1bad6e30d01bd9c920e8ee" dependencies = [ "monostate-impl", "serde", @@ -3699,20 +3730,26 @@ dependencies = [ [[package]] name = "monostate-impl" -version = "0.1.13" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" +checksum = "371717c0a5543d6a800cac822eac735aa7d2d2fbb41002e9856a4089532dbdce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] -name = "mutually_exclusive_features" -version = "0.1.0" +name = "mopa-maintained" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" +checksum = "79b7f3e22167862cc7c95b21a6f326c22e4bf40da59cbf000b368a310173ba11" + +[[package]] +name = "mutually_exclusive_features" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d02c0b00610773bb7fc61d85e13d86c7858cbdf00e1a120bfc41bc055dbaa0e" [[package]] name = "nohash" @@ -3762,19 +3799,20 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" dependencies = [ + "autocfg", "num-integer", "num-traits", ] [[package]] name = "num-complex" -version = "0.4.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" dependencies = [ "bytemuck", "num-traits", @@ -3788,18 +3826,19 @@ checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ + "autocfg", "num-traits", ] [[package]] name = "num-traits" -version = "0.2.19" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", "libm", @@ -3811,29 +3850,29 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", ] [[package]] name = "num_enum" -version = "0.7.3" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" dependencies = [ "num_enum_derive", ] [[package]] name = "num_enum_derive" -version = "0.7.3" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3853,9 +3892,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.5" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" dependencies = [ "memchr", ] @@ -3867,9 +3906,9 @@ source = "git+https://github.com/kerollmops/obkv?branch=unsized-kvreader#ce53587 [[package]] name = "once_cell" -version = "1.20.2" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "onig" @@ -3895,9 +3934,9 @@ dependencies = [ [[package]] name = "oorandom" -version = "11.1.4" +version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "option-ext" @@ -3907,9 +3946,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "ordered-float" -version = "4.4.0" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e7ccb95e240b7c9506a3d544f10d935e142cc90b0a1d56954fb44d89ad6b97" +checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be" dependencies = [ "num-traits", ] @@ -3948,22 +3987,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.3.5", "smallvec", - "windows-targets 0.52.6", + "windows-targets 0.48.1", ] [[package]] name = "paste" -version = "1.0.15" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "path-matchers" @@ -3992,11 +4031,11 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.4" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" +checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" dependencies = [ - "base64 0.22.1", + "base64 0.21.7", "serde", ] @@ -4016,20 +4055,19 @@ dependencies = [ [[package]] name = "pest" -version = "2.7.14" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" dependencies = [ - "memchr", "thiserror", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.14" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" +checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" dependencies = [ "pest", "pest_generator", @@ -4037,22 +4075,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.14" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" +checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] name = "pest_meta" -version = "2.7.14" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" +checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" dependencies = [ "once_cell", "pest", @@ -4099,7 +4137,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4113,29 +4151,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.7" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.7" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -4151,9 +4189,9 @@ checksum = "16f2611cd06a1ac239a0cea4521de9eb068a6ca110324ee00631aa68daa74fc0" [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "platform-dirs" @@ -4166,9 +4204,9 @@ dependencies = [ [[package]] name = "plotters" -version = "0.3.7" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" dependencies = [ "num-traits", "plotters-backend", @@ -4179,24 +4217,24 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.7" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" [[package]] name = "plotters-svg" -version = "0.3.7" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" dependencies = [ "plotters-backend", ] [[package]] name = "portable-atomic" -version = "1.9.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" [[package]] name = "powerfmt" @@ -4206,20 +4244,17 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" -dependencies = [ - "zerocopy", -] +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro-crate" -version = "3.2.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" dependencies = [ - "toml_edit", + "toml_edit 0.21.0", ] [[package]] @@ -4231,6 +4266,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", + "syn 1.0.109", "version_check", ] @@ -4322,9 +4358,9 @@ dependencies = [ [[package]] name = "pulp" -version = "0.18.22" +version = "0.18.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" +checksum = "03457ac216146f43f921500bac4e892d5cd32b0479b929cbfc90f95cd6c599c2" dependencies = [ "bytemuck", "libm", @@ -4334,17 +4370,16 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.0.0", + "rustc-hash 1.1.0", "rustls", - "socket2", "thiserror", "tokio", "tracing", @@ -4369,23 +4404,22 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.6" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e346e016eacfff12233c243718197ca12f148c84e1e84268a896699b41c71780" +checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46" dependencies = [ - "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.5.5", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] name = "quote" -version = "1.0.37" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -4439,12 +4473,12 @@ dependencies = [ [[package]] name = "raw-collections" version = "0.1.0" -source = "git+https://github.com/dureuill/raw-collections.git#48801130cb16d758ba9c610b0fe25747e4d85534" +source = "git+https://github.com/dureuill/raw-collections.git#e04a52424e1124ca63df66338a79c628e8f3bfd7" dependencies = [ "allocator-api2", "bitpacking", "bumpalo", - "hashbrown 0.15.0", + "hashbrown 0.15.1", "serde", "serde_json", ] @@ -4506,29 +4540,38 @@ checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags 2.6.0", + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", ] [[package]] name = "redox_users" -version = "0.4.6" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "libredox", + "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.11.1" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", @@ -4538,9 +4581,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", @@ -4555,9 +4598,9 @@ checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rend" @@ -4570,9 +4613,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ "base64 0.22.1", "bytes", @@ -4610,7 +4653,7 @@ dependencies = [ "wasm-streams", "web-sys", "webpki-roots", - "windows-registry", + "winreg", ] [[package]] @@ -4637,7 +4680,7 @@ source = "git+https://github.com/rhaiscript/rhai?rev=ef3df63121d27aacd838f366f2b dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4657,9 +4700,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.45" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" dependencies = [ "bitvec", "bytecheck", @@ -4675,9 +4718,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.45" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" dependencies = [ "proc-macro2", "quote", @@ -4708,9 +4751,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.36.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" +checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a" dependencies = [ "arrayvec", "borsh", @@ -4724,9 +4767,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustc-hash" @@ -4742,18 +4785,18 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.6.0", "errno", @@ -4764,9 +4807,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" dependencies = [ "log", "once_cell", @@ -4779,24 +4822,25 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.2.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ + "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "ring", "rustls-pki-types", @@ -4805,21 +4849,21 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "safetensors" -version = "0.4.5" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6" +checksum = "8d980e6bfb34436fb0a81e42bc41af43f11805bbbca443e7f68e9faaabe669ed" dependencies = [ "serde", "serde_json", @@ -4862,9 +4906,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" dependencies = [ "serde", ] @@ -4901,7 +4945,7 @@ checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4928,9 +4972,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.8" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -4997,9 +5041,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] @@ -5012,15 +5056,15 @@ checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "simdutf8" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "similar" -version = "2.6.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" +checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" [[package]] name = "simple_asn1" @@ -5048,9 +5092,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -5094,12 +5138,22 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", - "windows-sys 0.52.0", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", ] [[package]] @@ -5168,31 +5222,31 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.3" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] name = "subtle" -version = "2.6.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -5207,9 +5261,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -5225,7 +5279,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5233,9 +5287,6 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" -dependencies = [ - "futures-core", -] [[package]] name = "synchronoise" @@ -5254,7 +5305,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5294,9 +5345,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.42" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" dependencies = [ "filetime", "libc", @@ -5314,15 +5365,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.13.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", - "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5345,22 +5395,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5442,9 +5492,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -5487,31 +5537,32 @@ dependencies = [ [[package]] name = "tokio" -version = "1.41.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", "libc", "mio", + "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.5", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5527,9 +5578,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", @@ -5540,43 +5591,75 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.19" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.22.15", ] [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.22" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.22.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", - "winnow", + "winnow 0.6.13", ] [[package]] -name = "tower-service" -version = "0.3.3" +name = "tower" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" @@ -5592,9 +5675,9 @@ dependencies = [ [[package]] name = "tracing-actix-web" -version = "0.7.14" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b87073920bcce23e9f5cb0d2671e9f01d6803bb5229c159b2f5ce6806d73ffc" +checksum = "4ee9e39a66d9b615644893ffc1704d2a89b5b315b7fd0228ad3182ca9a306b19" dependencies = [ "actix-web", "mutually_exclusive_features", @@ -5611,7 +5694,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5690,9 +5773,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" @@ -5702,9 +5785,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "unescaper" @@ -5717,15 +5800,18 @@ dependencies = [ [[package]] name = "unicase" -version = "2.8.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-blocks" @@ -5735,15 +5821,15 @@ checksum = "6b12e05d9e06373163a9bb6bb8c263c261b396643a99445fe6b9811fd376581b" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.24" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] @@ -5759,15 +5845,15 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.14" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unicode_categories" @@ -5783,9 +5869,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.10.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" +checksum = "72139d247e5f97a3eff96229a7ae85ead5328a39efe76f8bf5a06313d505b6ea" dependencies = [ "base64 0.22.1", "flate2", @@ -5820,21 +5906,21 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "utf8-width" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" +checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" [[package]] name = "utf8parse" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.11.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", "serde", @@ -5854,24 +5940,24 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "9.0.1" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349ed9e45296a581f455bc18039878f409992999bc1d5da12a6800eb18c8752f" +checksum = "c32e7318e93a9ac53693b6caccfb05ff22e04a44c7cf8a279051f24c09da286f" dependencies = [ "anyhow", - "derive_builder 0.20.2", + "derive_builder 0.20.0", "rustversion", "vergen-lib", ] [[package]] name = "vergen-git2" -version = "1.0.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e771aff771c0d7c2f42e434e2766d304d917e29b40f0424e8faaaa936bbc3f29" +checksum = "a62c52cd2b2b8b7ec75fc20111b3022ac3ff83e4fc14b9497cfcfd39c54f9c67" dependencies = [ "anyhow", - "derive_builder 0.20.2", + "derive_builder 0.20.0", "git2", "rustversion", "time", @@ -5881,20 +5967,21 @@ dependencies = [ [[package]] name = "vergen-lib" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229eaddb0050920816cf051e619affaf18caa3dd512de8de5839ccbc8e53abb0" +checksum = "e06bee42361e43b60f363bad49d63798d0f42fb1768091812270eca00c784720" dependencies = [ "anyhow", - "derive_builder 0.20.2", + "derive_builder 0.20.0", + "getset", "rustversion", ] [[package]] name = "version_check" -version = "0.9.5" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" @@ -5934,35 +6021,34 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", - "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if", "js-sys", @@ -5972,9 +6058,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5982,28 +6068,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-streams" -version = "0.4.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -6014,9 +6100,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -6024,9 +6110,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" dependencies = [ "rustls-pki-types", ] @@ -6037,7 +6123,7 @@ version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "471d1c1645d361eb782a1650b1786a8fb58dd625e681a04c09f5ff7c8764a7b0" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.14.3", "once_cell", ] @@ -6059,11 +6145,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ - "windows-sys 0.59.0", + "winapi", ] [[package]] @@ -6079,7 +6165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core", - "windows-targets 0.52.6", + "windows-targets 0.52.4", ] [[package]] @@ -6088,37 +6174,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.6", + "windows-targets 0.52.4", ] [[package]] -name = "windows-registry" -version = "0.2.0" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-result", - "windows-strings", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-result" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-strings" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result", - "windows-targets 0.52.6", + "windows-targets 0.42.2", ] [[package]] @@ -6127,7 +6192,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.48.1", ] [[package]] @@ -6136,157 +6201,217 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", + "windows-targets 0.52.4", ] [[package]] name = "windows-targets" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] name = "windows-targets" -version = "0.52.6" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.6" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.52.6" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.52.6" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" [[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" +name = "windows_i686_gnu" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.52.6" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.52.6" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.6" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.52.6" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winnow" -version = "0.6.20" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] [[package]] -name = "wiremock" -version = "0.6.2" +name = "winnow" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fff469918e7ca034884c7fd8f93fe27bacb7fcb599fd879df6c7b429a29b646" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "wiremock" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec874e1eef0df2dcac546057fe5e29186f09c378181cd7b635b4b7bcc98e9d81" dependencies = [ "assert-json-diff", "async-trait", - "base64 0.22.1", + "base64 0.21.7", "deadpool", "futures", "http 1.1.0", @@ -6364,9 +6489,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.4" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "65e71b2e4f287f467794c671e2b8f8a5f3716b3c829079a1c44740148eff07e4" dependencies = [ "serde", "stable_deref_trait", @@ -6376,55 +6501,54 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "9e6936f0cce458098a201c245a11bef556c6a0181129c7034d10d76d1ec3a2b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "synstructure", ] [[package]] name = "zerocopy" -version = "0.7.35" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ - "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "655b0814c5c0b19ade497851070c640773304939a6c0fd5f5fb43da0696d05b7" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "e6a647510471d372f2e6c2e6b7219e44d8c574d24fdc11c610a61455782f18c3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "synstructure", ] @@ -6445,7 +6569,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -6465,9 +6589,9 @@ dependencies = [ [[package]] name = "zip" -version = "2.2.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" dependencies = [ "aes", "arbitrary", @@ -6517,18 +6641,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.1" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 1d25b9795..68e049f7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,24 +1,24 @@ [workspace] resolver = "2" members = [ - "meilisearch", - "meilitool", - "meilisearch-types", - "meilisearch-auth", - "meili-snap", - "index-scheduler", - "dump", - "file-store", - "permissive-json-pointer", - "milli", - "filter-parser", - "flatten-serde-json", - "json-depth-checker", - "benchmarks", - "fuzzers", - "tracing-trace", - "xtask", - "build-info", + "crates/meilisearch", + "crates/meilitool", + "crates/meilisearch-types", + "crates/meilisearch-auth", + "crates/meili-snap", + "crates/index-scheduler", + "crates/dump", + "crates/file-store", + "crates/permissive-json-pointer", + "crates/milli", + "crates/filter-parser", + "crates/flatten-serde-json", + "crates/json-depth-checker", + "crates/benchmarks", + "crates/fuzzers", + "crates/tracing-trace", + "crates/xtask", + "crates/build-info", ] [workspace.package] diff --git a/Dockerfile b/Dockerfile index 84d1da8f5..04557df59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ RUN set -eux; \ # Run FROM alpine:3.20 +LABEL org.opencontainers.image.source="https://github.com/meilisearch/meilisearch" ENV MEILI_HTTP_ADDR 0.0.0.0:7700 ENV MEILI_SERVER_PROVIDER docker diff --git a/README.md b/README.md index 59d618ab2..4be92d439 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@

- - + + + + +

diff --git a/assets/meilisearch-logo-kawaii.png b/assets/meilisearch-logo-kawaii.png deleted file mode 100644 index 40dc0cb0a..000000000 Binary files a/assets/meilisearch-logo-kawaii.png and /dev/null differ diff --git a/bors.toml b/bors.toml index 8750ed993..96e9ef65e 100644 --- a/bors.toml +++ b/bors.toml @@ -1,6 +1,6 @@ status = [ 'Tests on ubuntu-20.04', - 'Tests on macos-12', + 'Tests on macos-13', 'Tests on windows-2022', 'Run Clippy', 'Run Rustfmt', diff --git a/benchmarks/.gitignore b/crates/benchmarks/.gitignore similarity index 100% rename from benchmarks/.gitignore rename to crates/benchmarks/.gitignore diff --git a/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.toml similarity index 100% rename from benchmarks/Cargo.toml rename to crates/benchmarks/Cargo.toml diff --git a/benchmarks/README.md b/crates/benchmarks/README.md similarity index 100% rename from benchmarks/README.md rename to crates/benchmarks/README.md diff --git a/benchmarks/benches/indexing.rs b/crates/benchmarks/benches/indexing.rs similarity index 100% rename from benchmarks/benches/indexing.rs rename to crates/benchmarks/benches/indexing.rs diff --git a/benchmarks/benches/search_geo.rs b/crates/benchmarks/benches/search_geo.rs similarity index 100% rename from benchmarks/benches/search_geo.rs rename to crates/benchmarks/benches/search_geo.rs diff --git a/benchmarks/benches/search_songs.rs b/crates/benchmarks/benches/search_songs.rs similarity index 100% rename from benchmarks/benches/search_songs.rs rename to crates/benchmarks/benches/search_songs.rs diff --git a/benchmarks/benches/search_wiki.rs b/crates/benchmarks/benches/search_wiki.rs similarity index 100% rename from benchmarks/benches/search_wiki.rs rename to crates/benchmarks/benches/search_wiki.rs diff --git a/benchmarks/benches/utils.rs b/crates/benchmarks/benches/utils.rs similarity index 100% rename from benchmarks/benches/utils.rs rename to crates/benchmarks/benches/utils.rs diff --git a/benchmarks/build.rs b/crates/benchmarks/build.rs similarity index 100% rename from benchmarks/build.rs rename to crates/benchmarks/build.rs diff --git a/benchmarks/scripts/compare.sh b/crates/benchmarks/scripts/compare.sh similarity index 100% rename from benchmarks/scripts/compare.sh rename to crates/benchmarks/scripts/compare.sh diff --git a/benchmarks/scripts/list.sh b/crates/benchmarks/scripts/list.sh similarity index 100% rename from benchmarks/scripts/list.sh rename to crates/benchmarks/scripts/list.sh diff --git a/benchmarks/src/lib.rs b/crates/benchmarks/src/lib.rs similarity index 100% rename from benchmarks/src/lib.rs rename to crates/benchmarks/src/lib.rs diff --git a/build-info/Cargo.toml b/crates/build-info/Cargo.toml similarity index 100% rename from build-info/Cargo.toml rename to crates/build-info/Cargo.toml diff --git a/build-info/build.rs b/crates/build-info/build.rs similarity index 100% rename from build-info/build.rs rename to crates/build-info/build.rs diff --git a/build-info/src/lib.rs b/crates/build-info/src/lib.rs similarity index 100% rename from build-info/src/lib.rs rename to crates/build-info/src/lib.rs diff --git a/dump/Cargo.toml b/crates/dump/Cargo.toml similarity index 100% rename from dump/Cargo.toml rename to crates/dump/Cargo.toml diff --git a/dump/README.md b/crates/dump/README.md similarity index 100% rename from dump/README.md rename to crates/dump/README.md diff --git a/dump/src/error.rs b/crates/dump/src/error.rs similarity index 100% rename from dump/src/error.rs rename to crates/dump/src/error.rs diff --git a/dump/src/lib.rs b/crates/dump/src/lib.rs similarity index 100% rename from dump/src/lib.rs rename to crates/dump/src/lib.rs diff --git a/dump/src/reader/compat/mod.rs b/crates/dump/src/reader/compat/mod.rs similarity index 100% rename from dump/src/reader/compat/mod.rs rename to crates/dump/src/reader/compat/mod.rs diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-11.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-11.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-11.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-11.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-14.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-14.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-14.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-14.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-5.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-5.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-5.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-5.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-8.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-8.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-8.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-8.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-12.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-12.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-12.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-12.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-15.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-15.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-15.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-15.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-6.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-6.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-6.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-6.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-9.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-9.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-9.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-9.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-12.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-12.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-12.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-12.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-6.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-6.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-6.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-6.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-9.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-9.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-9.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-9.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-12.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-12.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-12.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-12.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-6.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-6.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-6.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-6.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-9.snap b/crates/dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-9.snap similarity index 100% rename from dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-9.snap rename to crates/dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-9.snap diff --git a/dump/src/reader/compat/v1_to_v2.rs b/crates/dump/src/reader/compat/v1_to_v2.rs similarity index 100% rename from dump/src/reader/compat/v1_to_v2.rs rename to crates/dump/src/reader/compat/v1_to_v2.rs diff --git a/dump/src/reader/compat/v2_to_v3.rs b/crates/dump/src/reader/compat/v2_to_v3.rs similarity index 100% rename from dump/src/reader/compat/v2_to_v3.rs rename to crates/dump/src/reader/compat/v2_to_v3.rs diff --git a/dump/src/reader/compat/v3_to_v4.rs b/crates/dump/src/reader/compat/v3_to_v4.rs similarity index 100% rename from dump/src/reader/compat/v3_to_v4.rs rename to crates/dump/src/reader/compat/v3_to_v4.rs diff --git a/dump/src/reader/compat/v4_to_v5.rs b/crates/dump/src/reader/compat/v4_to_v5.rs similarity index 100% rename from dump/src/reader/compat/v4_to_v5.rs rename to crates/dump/src/reader/compat/v4_to_v5.rs diff --git a/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs similarity index 100% rename from dump/src/reader/compat/v5_to_v6.rs rename to crates/dump/src/reader/compat/v5_to_v6.rs diff --git a/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs similarity index 100% rename from dump/src/reader/mod.rs rename to crates/dump/src/reader/mod.rs diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-10.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-10.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v1-10.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-10.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-4.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-4.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v1-4.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-4.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-7.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-7.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v1-7.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-7.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-11.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-11.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v2-11.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-11.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-14.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-14.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v2-14.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-14.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-5.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-5.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v2-5.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-5.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-8.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-8.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v2-8.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2-8.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-11.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-11.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-11.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-11.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-5.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-5.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-5.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-5.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-8.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-8.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-8.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-8.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-11.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-11.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v3-11.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-11.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-14.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-14.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v3-14.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-14.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-5.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-5.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v3-5.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-5.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-8.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-8.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v3-8.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v3-8.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v4-12.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v4-12.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v4-12.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v4-12.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v4-6.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v4-6.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v4-6.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v4-6.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v4-9.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v4-9.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v4-9.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v4-9.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v5-12.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v5-12.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v5-12.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v5-12.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v5-6.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v5-6.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v5-6.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v5-6.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v5-9.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v5-9.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v5-9.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v5-9.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-6.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-6.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-6.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-6.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-7.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-7.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-7.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-7.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-8.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-8.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-8.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-8.snap diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-9.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-9.snap similarity index 100% rename from dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-9.snap rename to crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-9.snap diff --git a/dump/src/reader/v1/mod.rs b/crates/dump/src/reader/v1/mod.rs similarity index 100% rename from dump/src/reader/v1/mod.rs rename to crates/dump/src/reader/v1/mod.rs diff --git a/dump/src/reader/v1/settings.rs b/crates/dump/src/reader/v1/settings.rs similarity index 100% rename from dump/src/reader/v1/settings.rs rename to crates/dump/src/reader/v1/settings.rs diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-10.snap b/crates/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-10.snap similarity index 100% rename from dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-10.snap rename to crates/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-10.snap diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-2.snap b/crates/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-2.snap similarity index 100% rename from dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-2.snap rename to crates/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-2.snap diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-6.snap b/crates/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-6.snap similarity index 100% rename from dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-6.snap rename to crates/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-6.snap diff --git a/dump/src/reader/v1/update.rs b/crates/dump/src/reader/v1/update.rs similarity index 100% rename from dump/src/reader/v1/update.rs rename to crates/dump/src/reader/v1/update.rs diff --git a/dump/src/reader/v2/errors.rs b/crates/dump/src/reader/v2/errors.rs similarity index 100% rename from dump/src/reader/v2/errors.rs rename to crates/dump/src/reader/v2/errors.rs diff --git a/dump/src/reader/v2/meta.rs b/crates/dump/src/reader/v2/meta.rs similarity index 100% rename from dump/src/reader/v2/meta.rs rename to crates/dump/src/reader/v2/meta.rs diff --git a/dump/src/reader/v2/mod.rs b/crates/dump/src/reader/v2/mod.rs similarity index 100% rename from dump/src/reader/v2/mod.rs rename to crates/dump/src/reader/v2/mod.rs diff --git a/dump/src/reader/v2/settings.rs b/crates/dump/src/reader/v2/settings.rs similarity index 100% rename from dump/src/reader/v2/settings.rs rename to crates/dump/src/reader/v2/settings.rs diff --git a/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-11.snap b/crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-11.snap similarity index 100% rename from dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-11.snap rename to crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-11.snap diff --git a/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-14.snap b/crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-14.snap similarity index 100% rename from dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-14.snap rename to crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-14.snap diff --git a/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-5.snap b/crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-5.snap similarity index 100% rename from dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-5.snap rename to crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-5.snap diff --git a/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-8.snap b/crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-8.snap similarity index 100% rename from dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-8.snap rename to crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2-8.snap diff --git a/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-10.snap b/crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-10.snap similarity index 100% rename from dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-10.snap rename to crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-10.snap diff --git a/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-4.snap b/crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-4.snap similarity index 100% rename from dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-4.snap rename to crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-4.snap diff --git a/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-7.snap b/crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-7.snap similarity index 100% rename from dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-7.snap rename to crates/dump/src/reader/v2/snapshots/dump__reader__v2__test__read_dump_v2_from_meilisearch_v0_22_0_issue_3435-7.snap diff --git a/dump/src/reader/v2/updates.rs b/crates/dump/src/reader/v2/updates.rs similarity index 100% rename from dump/src/reader/v2/updates.rs rename to crates/dump/src/reader/v2/updates.rs diff --git a/dump/src/reader/v3/errors.rs b/crates/dump/src/reader/v3/errors.rs similarity index 100% rename from dump/src/reader/v3/errors.rs rename to crates/dump/src/reader/v3/errors.rs diff --git a/dump/src/reader/v3/meta.rs b/crates/dump/src/reader/v3/meta.rs similarity index 100% rename from dump/src/reader/v3/meta.rs rename to crates/dump/src/reader/v3/meta.rs diff --git a/dump/src/reader/v3/mod.rs b/crates/dump/src/reader/v3/mod.rs similarity index 100% rename from dump/src/reader/v3/mod.rs rename to crates/dump/src/reader/v3/mod.rs diff --git a/dump/src/reader/v3/settings.rs b/crates/dump/src/reader/v3/settings.rs similarity index 100% rename from dump/src/reader/v3/settings.rs rename to crates/dump/src/reader/v3/settings.rs diff --git a/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-11.snap b/crates/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-11.snap similarity index 100% rename from dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-11.snap rename to crates/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-11.snap diff --git a/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-14.snap b/crates/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-14.snap similarity index 100% rename from dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-14.snap rename to crates/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-14.snap diff --git a/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-5.snap b/crates/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-5.snap similarity index 100% rename from dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-5.snap rename to crates/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-5.snap diff --git a/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-8.snap b/crates/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-8.snap similarity index 100% rename from dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-8.snap rename to crates/dump/src/reader/v3/snapshots/dump__reader__v3__test__read_dump_v3-8.snap diff --git a/dump/src/reader/v3/updates.rs b/crates/dump/src/reader/v3/updates.rs similarity index 100% rename from dump/src/reader/v3/updates.rs rename to crates/dump/src/reader/v3/updates.rs diff --git a/dump/src/reader/v4/errors.rs b/crates/dump/src/reader/v4/errors.rs similarity index 100% rename from dump/src/reader/v4/errors.rs rename to crates/dump/src/reader/v4/errors.rs diff --git a/dump/src/reader/v4/keys.rs b/crates/dump/src/reader/v4/keys.rs similarity index 100% rename from dump/src/reader/v4/keys.rs rename to crates/dump/src/reader/v4/keys.rs diff --git a/dump/src/reader/v4/meta.rs b/crates/dump/src/reader/v4/meta.rs similarity index 97% rename from dump/src/reader/v4/meta.rs rename to crates/dump/src/reader/v4/meta.rs index cec05f57c..2daea68a4 100644 --- a/dump/src/reader/v4/meta.rs +++ b/crates/dump/src/reader/v4/meta.rs @@ -74,7 +74,8 @@ impl Display for IndexUidFormatError { f, "invalid index uid `{}`, the uid must be an integer \ or a string containing only alphanumeric characters \ - a-z A-Z 0-9, hyphens - and underscores _.", + a-z A-Z 0-9, hyphens - and underscores _, \ + and can not be more than 400 bytes.", self.invalid_uid, ) } diff --git a/dump/src/reader/v4/mod.rs b/crates/dump/src/reader/v4/mod.rs similarity index 100% rename from dump/src/reader/v4/mod.rs rename to crates/dump/src/reader/v4/mod.rs diff --git a/dump/src/reader/v4/settings.rs b/crates/dump/src/reader/v4/settings.rs similarity index 100% rename from dump/src/reader/v4/settings.rs rename to crates/dump/src/reader/v4/settings.rs diff --git a/dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-10.snap b/crates/dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-10.snap similarity index 100% rename from dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-10.snap rename to crates/dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-10.snap diff --git a/dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-13.snap b/crates/dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-13.snap similarity index 100% rename from dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-13.snap rename to crates/dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-13.snap diff --git a/dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-7.snap b/crates/dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-7.snap similarity index 100% rename from dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-7.snap rename to crates/dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-7.snap diff --git a/dump/src/reader/v4/tasks.rs b/crates/dump/src/reader/v4/tasks.rs similarity index 100% rename from dump/src/reader/v4/tasks.rs rename to crates/dump/src/reader/v4/tasks.rs diff --git a/dump/src/reader/v5/errors.rs b/crates/dump/src/reader/v5/errors.rs similarity index 100% rename from dump/src/reader/v5/errors.rs rename to crates/dump/src/reader/v5/errors.rs diff --git a/dump/src/reader/v5/keys.rs b/crates/dump/src/reader/v5/keys.rs similarity index 100% rename from dump/src/reader/v5/keys.rs rename to crates/dump/src/reader/v5/keys.rs diff --git a/dump/src/reader/v5/meta.rs b/crates/dump/src/reader/v5/meta.rs similarity index 97% rename from dump/src/reader/v5/meta.rs rename to crates/dump/src/reader/v5/meta.rs index cec05f57c..2daea68a4 100644 --- a/dump/src/reader/v5/meta.rs +++ b/crates/dump/src/reader/v5/meta.rs @@ -74,7 +74,8 @@ impl Display for IndexUidFormatError { f, "invalid index uid `{}`, the uid must be an integer \ or a string containing only alphanumeric characters \ - a-z A-Z 0-9, hyphens - and underscores _.", + a-z A-Z 0-9, hyphens - and underscores _, \ + and can not be more than 400 bytes.", self.invalid_uid, ) } diff --git a/dump/src/reader/v5/mod.rs b/crates/dump/src/reader/v5/mod.rs similarity index 100% rename from dump/src/reader/v5/mod.rs rename to crates/dump/src/reader/v5/mod.rs diff --git a/dump/src/reader/v5/settings.rs b/crates/dump/src/reader/v5/settings.rs similarity index 100% rename from dump/src/reader/v5/settings.rs rename to crates/dump/src/reader/v5/settings.rs diff --git a/dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-10.snap b/crates/dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-10.snap similarity index 100% rename from dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-10.snap rename to crates/dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-10.snap diff --git a/dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-13.snap b/crates/dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-13.snap similarity index 100% rename from dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-13.snap rename to crates/dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-13.snap diff --git a/dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-7.snap b/crates/dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-7.snap similarity index 100% rename from dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-7.snap rename to crates/dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-7.snap diff --git a/dump/src/reader/v5/tasks.rs b/crates/dump/src/reader/v5/tasks.rs similarity index 100% rename from dump/src/reader/v5/tasks.rs rename to crates/dump/src/reader/v5/tasks.rs diff --git a/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs similarity index 100% rename from dump/src/reader/v6/mod.rs rename to crates/dump/src/reader/v6/mod.rs diff --git a/dump/src/writer.rs b/crates/dump/src/writer.rs similarity index 100% rename from dump/src/writer.rs rename to crates/dump/src/writer.rs diff --git a/dump/tests/assets/v1.dump b/crates/dump/tests/assets/v1.dump similarity index 100% rename from dump/tests/assets/v1.dump rename to crates/dump/tests/assets/v1.dump diff --git a/dump/tests/assets/v2-v0.22.0.dump b/crates/dump/tests/assets/v2-v0.22.0.dump similarity index 100% rename from dump/tests/assets/v2-v0.22.0.dump rename to crates/dump/tests/assets/v2-v0.22.0.dump diff --git a/dump/tests/assets/v2.dump b/crates/dump/tests/assets/v2.dump similarity index 100% rename from dump/tests/assets/v2.dump rename to crates/dump/tests/assets/v2.dump diff --git a/dump/tests/assets/v3.dump b/crates/dump/tests/assets/v3.dump similarity index 100% rename from dump/tests/assets/v3.dump rename to crates/dump/tests/assets/v3.dump diff --git a/dump/tests/assets/v4.dump b/crates/dump/tests/assets/v4.dump similarity index 100% rename from dump/tests/assets/v4.dump rename to crates/dump/tests/assets/v4.dump diff --git a/dump/tests/assets/v5.dump b/crates/dump/tests/assets/v5.dump similarity index 100% rename from dump/tests/assets/v5.dump rename to crates/dump/tests/assets/v5.dump diff --git a/dump/tests/assets/v6-with-experimental.dump b/crates/dump/tests/assets/v6-with-experimental.dump similarity index 100% rename from dump/tests/assets/v6-with-experimental.dump rename to crates/dump/tests/assets/v6-with-experimental.dump diff --git a/dump/tests/assets/v6-with-vectors.dump b/crates/dump/tests/assets/v6-with-vectors.dump similarity index 100% rename from dump/tests/assets/v6-with-vectors.dump rename to crates/dump/tests/assets/v6-with-vectors.dump diff --git a/file-store/Cargo.toml b/crates/file-store/Cargo.toml similarity index 100% rename from file-store/Cargo.toml rename to crates/file-store/Cargo.toml diff --git a/file-store/src/lib.rs b/crates/file-store/src/lib.rs similarity index 100% rename from file-store/src/lib.rs rename to crates/file-store/src/lib.rs diff --git a/filter-parser/Cargo.toml b/crates/filter-parser/Cargo.toml similarity index 100% rename from filter-parser/Cargo.toml rename to crates/filter-parser/Cargo.toml diff --git a/filter-parser/README.md b/crates/filter-parser/README.md similarity index 100% rename from filter-parser/README.md rename to crates/filter-parser/README.md diff --git a/filter-parser/fuzz/.gitignore b/crates/filter-parser/fuzz/.gitignore similarity index 100% rename from filter-parser/fuzz/.gitignore rename to crates/filter-parser/fuzz/.gitignore diff --git a/filter-parser/fuzz/Cargo.toml b/crates/filter-parser/fuzz/Cargo.toml similarity index 100% rename from filter-parser/fuzz/Cargo.toml rename to crates/filter-parser/fuzz/Cargo.toml diff --git a/filter-parser/fuzz/corpus/parse/test_1 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_1 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_1 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_1 diff --git a/filter-parser/fuzz/corpus/parse/test_10 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_10 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_10 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_10 diff --git a/filter-parser/fuzz/corpus/parse/test_11 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_11 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_11 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_11 diff --git a/filter-parser/fuzz/corpus/parse/test_12 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_12 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_12 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_12 diff --git a/filter-parser/fuzz/corpus/parse/test_13 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_13 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_13 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_13 diff --git a/filter-parser/fuzz/corpus/parse/test_14 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_14 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_14 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_14 diff --git a/filter-parser/fuzz/corpus/parse/test_15 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_15 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_15 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_15 diff --git a/filter-parser/fuzz/corpus/parse/test_16 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_16 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_16 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_16 diff --git a/filter-parser/fuzz/corpus/parse/test_17 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_17 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_17 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_17 diff --git a/filter-parser/fuzz/corpus/parse/test_18 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_18 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_18 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_18 diff --git a/filter-parser/fuzz/corpus/parse/test_19 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_19 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_19 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_19 diff --git a/filter-parser/fuzz/corpus/parse/test_2 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_2 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_2 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_2 diff --git a/filter-parser/fuzz/corpus/parse/test_20 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_20 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_20 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_20 diff --git a/filter-parser/fuzz/corpus/parse/test_21 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_21 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_21 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_21 diff --git a/filter-parser/fuzz/corpus/parse/test_22 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_22 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_22 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_22 diff --git a/filter-parser/fuzz/corpus/parse/test_23 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_23 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_23 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_23 diff --git a/filter-parser/fuzz/corpus/parse/test_24 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_24 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_24 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_24 diff --git a/filter-parser/fuzz/corpus/parse/test_25 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_25 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_25 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_25 diff --git a/filter-parser/fuzz/corpus/parse/test_26 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_26 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_26 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_26 diff --git a/filter-parser/fuzz/corpus/parse/test_27 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_27 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_27 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_27 diff --git a/filter-parser/fuzz/corpus/parse/test_28 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_28 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_28 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_28 diff --git a/filter-parser/fuzz/corpus/parse/test_29 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_29 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_29 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_29 diff --git a/filter-parser/fuzz/corpus/parse/test_3 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_3 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_3 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_3 diff --git a/filter-parser/fuzz/corpus/parse/test_30 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_30 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_30 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_30 diff --git a/filter-parser/fuzz/corpus/parse/test_31 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_31 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_31 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_31 diff --git a/filter-parser/fuzz/corpus/parse/test_32 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_32 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_32 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_32 diff --git a/filter-parser/fuzz/corpus/parse/test_33 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_33 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_33 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_33 diff --git a/filter-parser/fuzz/corpus/parse/test_34 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_34 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_34 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_34 diff --git a/filter-parser/fuzz/corpus/parse/test_35 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_35 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_35 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_35 diff --git a/filter-parser/fuzz/corpus/parse/test_36 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_36 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_36 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_36 diff --git a/filter-parser/fuzz/corpus/parse/test_37 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_37 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_37 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_37 diff --git a/filter-parser/fuzz/corpus/parse/test_38 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_38 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_38 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_38 diff --git a/filter-parser/fuzz/corpus/parse/test_39 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_39 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_39 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_39 diff --git a/filter-parser/fuzz/corpus/parse/test_4 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_4 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_4 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_4 diff --git a/filter-parser/fuzz/corpus/parse/test_40 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_40 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_40 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_40 diff --git a/filter-parser/fuzz/corpus/parse/test_41 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_41 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_41 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_41 diff --git a/filter-parser/fuzz/corpus/parse/test_42 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_42 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_42 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_42 diff --git a/filter-parser/fuzz/corpus/parse/test_43 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_43 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_43 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_43 diff --git a/filter-parser/fuzz/corpus/parse/test_5 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_5 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_5 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_5 diff --git a/filter-parser/fuzz/corpus/parse/test_6 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_6 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_6 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_6 diff --git a/filter-parser/fuzz/corpus/parse/test_7 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_7 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_7 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_7 diff --git a/filter-parser/fuzz/corpus/parse/test_8 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_8 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_8 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_8 diff --git a/filter-parser/fuzz/corpus/parse/test_9 b/crates/filter-parser/fuzz/fuzz/corpus/parse/test_9 similarity index 100% rename from filter-parser/fuzz/corpus/parse/test_9 rename to crates/filter-parser/fuzz/fuzz/corpus/parse/test_9 diff --git a/filter-parser/fuzz/fuzz_targets/parse.rs b/crates/filter-parser/fuzz/fuzz_targets/parse.rs similarity index 100% rename from filter-parser/fuzz/fuzz_targets/parse.rs rename to crates/filter-parser/fuzz/fuzz_targets/parse.rs diff --git a/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs similarity index 100% rename from filter-parser/src/condition.rs rename to crates/filter-parser/src/condition.rs diff --git a/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs similarity index 100% rename from filter-parser/src/error.rs rename to crates/filter-parser/src/error.rs diff --git a/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs similarity index 100% rename from filter-parser/src/lib.rs rename to crates/filter-parser/src/lib.rs diff --git a/filter-parser/src/main.rs b/crates/filter-parser/src/main.rs similarity index 100% rename from filter-parser/src/main.rs rename to crates/filter-parser/src/main.rs diff --git a/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs similarity index 100% rename from filter-parser/src/value.rs rename to crates/filter-parser/src/value.rs diff --git a/flatten-serde-json/Cargo.toml b/crates/flatten-serde-json/Cargo.toml similarity index 100% rename from flatten-serde-json/Cargo.toml rename to crates/flatten-serde-json/Cargo.toml diff --git a/flatten-serde-json/README.md b/crates/flatten-serde-json/README.md similarity index 100% rename from flatten-serde-json/README.md rename to crates/flatten-serde-json/README.md diff --git a/flatten-serde-json/benches/benchmarks.rs b/crates/flatten-serde-json/benches/benchmarks.rs similarity index 100% rename from flatten-serde-json/benches/benchmarks.rs rename to crates/flatten-serde-json/benches/benchmarks.rs diff --git a/flatten-serde-json/fuzz/Cargo.toml b/crates/flatten-serde-json/fuzz/Cargo.toml similarity index 100% rename from flatten-serde-json/fuzz/Cargo.toml rename to crates/flatten-serde-json/fuzz/Cargo.toml diff --git a/flatten-serde-json/fuzz/fuzz_targets/flatten.rs b/crates/flatten-serde-json/fuzz/fuzz_targets/flatten.rs similarity index 100% rename from flatten-serde-json/fuzz/fuzz_targets/flatten.rs rename to crates/flatten-serde-json/fuzz/fuzz_targets/flatten.rs diff --git a/flatten-serde-json/src/lib.rs b/crates/flatten-serde-json/src/lib.rs similarity index 100% rename from flatten-serde-json/src/lib.rs rename to crates/flatten-serde-json/src/lib.rs diff --git a/flatten-serde-json/src/main.rs b/crates/flatten-serde-json/src/main.rs similarity index 100% rename from flatten-serde-json/src/main.rs rename to crates/flatten-serde-json/src/main.rs diff --git a/fuzzers/Cargo.toml b/crates/fuzzers/Cargo.toml similarity index 100% rename from fuzzers/Cargo.toml rename to crates/fuzzers/Cargo.toml diff --git a/fuzzers/README.md b/crates/fuzzers/README.md similarity index 100% rename from fuzzers/README.md rename to crates/fuzzers/README.md diff --git a/fuzzers/src/bin/fuzz-indexing.rs b/crates/fuzzers/src/bin/fuzz-indexing.rs similarity index 100% rename from fuzzers/src/bin/fuzz-indexing.rs rename to crates/fuzzers/src/bin/fuzz-indexing.rs diff --git a/fuzzers/src/lib.rs b/crates/fuzzers/src/lib.rs similarity index 100% rename from fuzzers/src/lib.rs rename to crates/fuzzers/src/lib.rs diff --git a/index-scheduler/Cargo.toml b/crates/index-scheduler/Cargo.toml similarity index 92% rename from index-scheduler/Cargo.toml rename to crates/index-scheduler/Cargo.toml index 88f9488b5..4a2913083 100644 --- a/index-scheduler/Cargo.toml +++ b/crates/index-scheduler/Cargo.toml @@ -42,7 +42,7 @@ uuid = { version = "1.10.0", features = ["serde", "v4"] } bumpalo = "3.16.0" [dev-dependencies] -arroy = { git = "https://github.com/meilisearch/arroy/", rev = "2386594dfb009ce08821a925ccc89fb8e30bf73d" } +arroy = "0.5.0" big_s = "1.0.2" crossbeam = "0.8.4" insta = { version = "1.39.0", features = ["json", "redactions"] } diff --git a/index-scheduler/src/autobatcher.rs b/crates/index-scheduler/src/autobatcher.rs similarity index 100% rename from index-scheduler/src/autobatcher.rs rename to crates/index-scheduler/src/autobatcher.rs diff --git a/index-scheduler/src/batch.rs b/crates/index-scheduler/src/batch.rs similarity index 100% rename from index-scheduler/src/batch.rs rename to crates/index-scheduler/src/batch.rs diff --git a/index-scheduler/src/error.rs b/crates/index-scheduler/src/error.rs similarity index 99% rename from index-scheduler/src/error.rs rename to crates/index-scheduler/src/error.rs index 223b84762..3bd378fd6 100644 --- a/index-scheduler/src/error.rs +++ b/crates/index-scheduler/src/error.rs @@ -101,7 +101,7 @@ pub enum Error { )] InvalidTaskCanceledBy { canceled_by: String }, #[error( - "{index_uid} is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_)." + "{index_uid} is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes." )] InvalidIndexUid { index_uid: String }, #[error("Task `{0}` not found.")] diff --git a/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs similarity index 100% rename from index-scheduler/src/features.rs rename to crates/index-scheduler/src/features.rs diff --git a/index-scheduler/src/index_mapper/index_map.rs b/crates/index-scheduler/src/index_mapper/index_map.rs similarity index 100% rename from index-scheduler/src/index_mapper/index_map.rs rename to crates/index-scheduler/src/index_mapper/index_map.rs diff --git a/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs similarity index 100% rename from index-scheduler/src/index_mapper/mod.rs rename to crates/index-scheduler/src/index_mapper/mod.rs diff --git a/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs similarity index 100% rename from index-scheduler/src/insta_snapshot.rs rename to crates/index-scheduler/src/insta_snapshot.rs diff --git a/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs similarity index 99% rename from index-scheduler/src/lib.rs rename to crates/index-scheduler/src/lib.rs index fe8244f9b..e0e2bfb75 100644 --- a/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -1263,7 +1263,7 @@ impl IndexScheduler { #[cfg(test)] self.maybe_fail(tests::FailureLocation::UpdatingTaskAfterProcessBatchFailure)?; - tracing::info!("Batch failed {}", error); + tracing::error!("Batch failed {}", error); self.update_task(&mut wtxn, &task) .map_err(|e| Error::TaskDatabaseUpdate(Box::new(e)))?; diff --git a/index-scheduler/src/lru.rs b/crates/index-scheduler/src/lru.rs similarity index 100% rename from index-scheduler/src/lru.rs rename to crates/index-scheduler/src/lru.rs diff --git a/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-15.snap b/crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-15.snap similarity index 100% rename from index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-15.snap rename to crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-15.snap diff --git a/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-2.snap b/crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-2.snap similarity index 100% rename from index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-2.snap rename to crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-2.snap diff --git a/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-22.snap b/crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-22.snap similarity index 100% rename from index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-22.snap rename to crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-22.snap diff --git a/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-5.snap b/crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-5.snap similarity index 100% rename from index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-5.snap rename to crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-5.snap diff --git a/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-8.snap b/crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-8.snap similarity index 100% rename from index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-8.snap rename to crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-8.snap diff --git a/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors.snap b/crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors.snap similarity index 100% rename from index-scheduler/src/snapshots/index_scheduler__tests__import_vectors.snap rename to crates/index-scheduler/src/snapshots/index_scheduler__tests__import_vectors.snap diff --git a/index-scheduler/src/snapshots/index_scheduler__tests__settings_update-2.snap b/crates/index-scheduler/src/snapshots/index_scheduler__tests__settings_update-2.snap similarity index 100% rename from index-scheduler/src/snapshots/index_scheduler__tests__settings_update-2.snap rename to crates/index-scheduler/src/snapshots/index_scheduler__tests__settings_update-2.snap diff --git a/index-scheduler/src/snapshots/index_scheduler__tests__settings_update-5.snap b/crates/index-scheduler/src/snapshots/index_scheduler__tests__settings_update-5.snap similarity index 100% rename from index-scheduler/src/snapshots/index_scheduler__tests__settings_update-5.snap rename to crates/index-scheduler/src/snapshots/index_scheduler__tests__settings_update-5.snap diff --git a/index-scheduler/src/snapshots/index_scheduler__tests__settings_update.snap b/crates/index-scheduler/src/snapshots/index_scheduler__tests__settings_update.snap similarity index 100% rename from index-scheduler/src/snapshots/index_scheduler__tests__settings_update.snap rename to crates/index-scheduler/src/snapshots/index_scheduler__tests__settings_update.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/initial_tasks_enqueued.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/initial_tasks_enqueued.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/initial_tasks_enqueued.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/initial_tasks_enqueued.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/aborted_indexation.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/aborted_indexation.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/aborted_indexation.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/aborted_indexation.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/first_task_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/first_task_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/first_task_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/first_task_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/processing_second_task_cancel_enqueued.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/processing_second_task_cancel_enqueued.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/processing_second_task_cancel_enqueued.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/processing_second_task_cancel_enqueued.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/after_dump_register.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/after_dump_register.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/after_dump_register.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/after_dump_register.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_registered.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_registered.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_registered.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_dump/cancel_registered.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/aborted_indexation.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/aborted_indexation.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_processing_task/aborted_indexation.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/aborted_indexation.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_task_registered.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_task_registered.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_task_registered.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_task_registered.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/initial_task_processing.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/initial_task_processing.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_processing_task/initial_task_processing.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/initial_task_processing.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_processing_task/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_processing_task/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/cancel_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/cancel_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/cancel_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/cancel_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/initial_task_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/initial_task_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/initial_task_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/initial_task_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition/after_register.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition/after_register.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition/after_register.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition/after_register.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition/after_the_batch_creation.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition/after_the_batch_creation.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition/after_the_batch_creation.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition/after_the_batch_creation.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition/once_everything_is_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition/once_everything_is_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition/once_everything_is_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition/once_everything_is_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/before_index_creation.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/before_index_creation.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/before_index_creation.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/before_index_creation.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/both_task_succeeded.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/both_task_succeeded.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/both_task_succeeded.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/both_task_succeeded.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_second_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_second_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_second_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_second_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_third_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_third_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_third_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/registered_the_third_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion_on_unexisting_index/1.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion_on_unexisting_index/1.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion_on_unexisting_index/1.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion_on_unexisting_index/1.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion_on_unexisting_index/2.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion_on_unexisting_index/2.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion_on_unexisting_index/2.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion_on_unexisting_index/2.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_batch_created.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_batch_created.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_batch_created.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_batch_created.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings_and_documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings_and_documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings_and_documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings_and_documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/documents_remaining_should_only_be_bork.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/documents_remaining_should_only_be_bork.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/documents_remaining_should_only_be_bork.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/documents_remaining_should_only_be_bork.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/after_register.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/after_register.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/after_register.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/after_register.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/after_batch_succeeded.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/after_batch_succeeded.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/after_batch_succeeded.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/after_batch_succeeded.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/after_failing_to_commit.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/after_failing_to_commit.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/after_failing_to_commit.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/after_failing_to_commit.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/document_addition_succeeded_but_index_scheduler_not_updated.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/document_addition_succeeded_but_index_scheduler_not_updated.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/document_addition_succeeded_but_index_scheduler_not_updated.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/document_addition_succeeded_but_index_scheduler_not_updated.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/task_successfully_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/task_successfully_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/task_successfully_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/fail_in_update_task_after_process_batch_success_for_document_addition/task_successfully_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir succeeds.snap b/crates/index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir succeeds.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir succeeds.snap rename to crates/index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir succeeds.snap diff --git a/index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir.snap b/crates/index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir.snap rename to crates/index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir.snap diff --git a/index-scheduler/src/snapshots/lib.rs/import_vectors/adding Intel succeeds.snap b/crates/index-scheduler/src/snapshots/lib.rs/import_vectors/adding Intel succeeds.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/import_vectors/adding Intel succeeds.snap rename to crates/index-scheduler/src/snapshots/lib.rs/import_vectors/adding Intel succeeds.snap diff --git a/index-scheduler/src/snapshots/lib.rs/import_vectors/after adding Intel.snap b/crates/index-scheduler/src/snapshots/lib.rs/import_vectors/after adding Intel.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/import_vectors/after adding Intel.snap rename to crates/index-scheduler/src/snapshots/lib.rs/import_vectors/after adding Intel.snap diff --git a/index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap b/crates/index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap rename to crates/index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap diff --git a/index-scheduler/src/snapshots/lib.rs/import_vectors/settings_update_processed_vectors.snap b/crates/index-scheduler/src/snapshots/lib.rs/import_vectors/settings_update_processed_vectors.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/import_vectors/settings_update_processed_vectors.snap rename to crates/index-scheduler/src/snapshots/lib.rs/import_vectors/settings_update_processed_vectors.snap diff --git a/index-scheduler/src/snapshots/lib.rs/import_vectors_first_and_embedder_later/documents after initial push.snap b/crates/index-scheduler/src/snapshots/lib.rs/import_vectors_first_and_embedder_later/documents after initial push.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/import_vectors_first_and_embedder_later/documents after initial push.snap rename to crates/index-scheduler/src/snapshots/lib.rs/import_vectors_first_and_embedder_later/documents after initial push.snap diff --git a/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap b/crates/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap rename to crates/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap diff --git a/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/first.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/first.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/first.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/first.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/fourth.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/fourth.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/fourth.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/fourth.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_second_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_second_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_second_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_second_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_third_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_third_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_third_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/registered_the_third_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/second.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/second.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/second.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/second.snap diff --git a/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/third.snap b/crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/third.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/third.snap rename to crates/index-scheduler/src/snapshots/lib.rs/process_tasks_without_autobatching/third.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap rename to crates/index-scheduler/src/snapshots/lib.rs/query_tasks_canceled_by/start.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/processed_all_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/processed_all_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/processed_all_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_second_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_second_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_second_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_third_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_third_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/query_tasks_from_and_limit/registered_the_third_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap rename to crates/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/start.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/start.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_tasks_simple/start.snap rename to crates/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/start.snap diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_special_rules/start.snap b/crates/index-scheduler/src/snapshots/lib.rs/query_tasks_special_rules/start.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/query_tasks_special_rules/start.snap rename to crates/index-scheduler/src/snapshots/lib.rs/query_tasks_special_rules/start.snap diff --git a/index-scheduler/src/snapshots/lib.rs/register/everything_is_successfully_registered.snap b/crates/index-scheduler/src/snapshots/lib.rs/register/everything_is_successfully_registered.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/register/everything_is_successfully_registered.snap rename to crates/index-scheduler/src/snapshots/lib.rs/register/everything_is_successfully_registered.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_a.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_a.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes/create_a.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_a.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_b.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_b.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes/create_b.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_b.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_c.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_c.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes/create_c.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_c.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_d.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_d.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes/create_d.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/create_d.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes/first_swap_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/first_swap_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes/first_swap_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/first_swap_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes/first_swap_registered.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/first_swap_registered.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes/first_swap_registered.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/first_swap_registered.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes/second_swap_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/second_swap_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes/second_swap_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/second_swap_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes/third_empty_swap_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/third_empty_swap_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes/third_empty_swap_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/third_empty_swap_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes/two_swaps_registered.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/two_swaps_registered.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes/two_swaps_registered.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes/two_swaps_registered.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/after_the_index_creation.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/after_the_index_creation.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/after_the_index_creation.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/after_the_index_creation.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/first_swap_failed.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/first_swap_failed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/first_swap_failed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/first_swap_failed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/initial_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/initial_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/initial_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/swap_indexes_errors/initial_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_enqueued.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_enqueued.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_enqueued.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_enqueued.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/initial_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/task_deletion_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/task_deletion_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/task_deletion_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/task_deletion_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/after_registering_the_task_deletion.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/after_registering_the_task_deletion.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/after_registering_the_task_deletion.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/after_registering_the_task_deletion.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_enqueued.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_enqueued.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_enqueued.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_enqueued.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/initial_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/task_deletion_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/task_deletion_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/task_deletion_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_deleteable/task_deletion_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_done.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_enqueued.snap diff --git a/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap b/crates/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap rename to crates/index-scheduler/src/snapshots/lib.rs/task_deletion_undeleteable/task_deletion_processing.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/everything_has_been_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/everything_has_been_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/everything_has_been_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/everything_has_been_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_processing_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_processing_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_processing_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_processing_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_registering_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_registering_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index/after_registering_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/after_registering_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/after_registering_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/after_registering_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/all_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/all_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/all_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/all_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/five_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/five_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/five_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_cant_create_index_without_index_without_autobatching/five_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/after_registering_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/after_registering_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/after_registering_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/all_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/all_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/all_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/all_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/only_first_task_failed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/only_first_task_failed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/only_first_task_failed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_right_without_index_starts_with_cant_create/only_first_task_failed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/after_registering_the_5_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/after_registering_the_5_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/after_registering_the_5_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/after_registering_the_5_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fifth_task_succeeds.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fifth_task_succeeds.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fifth_task_succeeds.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fifth_task_succeeds.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/first_and_second_task_fails.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/first_and_second_task_fails.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/first_and_second_task_fails.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/first_and_second_task_fails.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fourth_task_fails.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fourth_task_fails.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fourth_task_fails.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/fourth_task_fails.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/third_task_succeeds.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/third_task_succeeds.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/third_task_succeeds.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_bad_primary_key/third_task_succeeds.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/after_registering_the_3_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/after_registering_the_3_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/after_registering_the_3_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/after_registering_the_3_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/only_first_task_succeed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/only_first_task_succeed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/only_first_task_succeed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/only_first_task_succeed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/second_task_fails.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/second_task_fails.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/second_task_fails.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/second_task_fails.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/third_task_fails.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/third_task_fails.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/third_task_fails.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key/third_task_fails.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/after_registering_the_3_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/after_registering_the_3_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/after_registering_the_3_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/after_registering_the_3_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/only_first_task_succeed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/only_first_task_succeed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/only_first_task_succeed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/only_first_task_succeed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/second_and_third_tasks_fails.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/second_and_third_tasks_fails.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/second_and_third_tasks_fails.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_multiple_primary_key_batch_wrong_key/second_and_third_tasks_fails.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/after_registering_the_6_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/after_registering_the_6_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/after_registering_the_6_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/after_registering_the_6_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/all_other_tasks_succeeds.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/all_other_tasks_succeeds.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/all_other_tasks_succeeds.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/all_other_tasks_succeeds.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/first_task_fails.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/first_task_fails.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/first_task_fails.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/first_task_fails.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/second_task_fails.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/second_task_fails.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/second_task_fails.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/second_task_fails.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/third_task_succeeds.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/third_task_succeeds.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/third_task_succeeds.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key/third_task_succeeds.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/after_registering_the_6_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/after_registering_the_6_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/after_registering_the_6_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/after_registering_the_6_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/all_other_tasks_succeeds.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/all_other_tasks_succeeds.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/all_other_tasks_succeeds.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/all_other_tasks_succeeds.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/first_task_succeed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/first_task_succeed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/first_task_succeed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/first_task_succeed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/second_task_fails.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/second_task_fails.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/second_task_fails.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/second_task_fails.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/third_task_succeeds.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/third_task_succeeds.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/third_task_succeeds.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_addition_with_set_and_null_primary_key_inference_works/third_task_succeeds.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_replace/1.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_replace/1.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_replace/1.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_replace/1.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_replace/2.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_replace/2.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_replace/2.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_replace/2.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_replace/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_replace/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_replace/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_replace/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/after_registering_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/after_registering_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/after_registering_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/all_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/all_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/all_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/all_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/five_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/five_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/five_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_replace_without_autobatching/five_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_update/1.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_update/1.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_update/1.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_update/1.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_update/2.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_update/2.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_update/2.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_update/2.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_update/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_update/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_update/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_update/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/after_registering_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/after_registering_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/after_registering_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/all_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/all_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/all_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/all_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/five_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/five_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/five_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_document_update_without_autobatching/five_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/after_registering_the_10_tasks.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/after_registering_the_10_tasks.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/after_registering_the_10_tasks.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/all_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/all_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/all_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/all_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/documents.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/documents.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/documents.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/documents.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/five_tasks_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/five_tasks_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/five_tasks_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_mixed_document_addition/five_tasks_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_settings_update/settings_update_processed.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_settings_update/settings_update_processed.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_settings_update/settings_update_processed.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_settings_update/settings_update_processed.snap diff --git a/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap b/crates/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap similarity index 100% rename from index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap rename to crates/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap diff --git a/index-scheduler/src/utils.rs b/crates/index-scheduler/src/utils.rs similarity index 100% rename from index-scheduler/src/utils.rs rename to crates/index-scheduler/src/utils.rs diff --git a/index-scheduler/src/uuid_codec.rs b/crates/index-scheduler/src/uuid_codec.rs similarity index 100% rename from index-scheduler/src/uuid_codec.rs rename to crates/index-scheduler/src/uuid_codec.rs diff --git a/json-depth-checker/Cargo.toml b/crates/json-depth-checker/Cargo.toml similarity index 100% rename from json-depth-checker/Cargo.toml rename to crates/json-depth-checker/Cargo.toml diff --git a/json-depth-checker/benches/depth.rs b/crates/json-depth-checker/benches/depth.rs similarity index 100% rename from json-depth-checker/benches/depth.rs rename to crates/json-depth-checker/benches/depth.rs diff --git a/json-depth-checker/fuzz/Cargo.toml b/crates/json-depth-checker/fuzz/Cargo.toml similarity index 100% rename from json-depth-checker/fuzz/Cargo.toml rename to crates/json-depth-checker/fuzz/Cargo.toml diff --git a/json-depth-checker/fuzz/fuzz_targets/depth.rs b/crates/json-depth-checker/fuzz/fuzz_targets/depth.rs similarity index 100% rename from json-depth-checker/fuzz/fuzz_targets/depth.rs rename to crates/json-depth-checker/fuzz/fuzz_targets/depth.rs diff --git a/json-depth-checker/src/lib.rs b/crates/json-depth-checker/src/lib.rs similarity index 100% rename from json-depth-checker/src/lib.rs rename to crates/json-depth-checker/src/lib.rs diff --git a/meili-snap/Cargo.toml b/crates/meili-snap/Cargo.toml similarity index 100% rename from meili-snap/Cargo.toml rename to crates/meili-snap/Cargo.toml diff --git a/meili-snap/src/lib.rs b/crates/meili-snap/src/lib.rs similarity index 100% rename from meili-snap/src/lib.rs rename to crates/meili-snap/src/lib.rs diff --git a/meili-snap/src/snapshots/lib.rs/snap/4.snap b/crates/meili-snap/src/snapshots/lib.rs/snap/4.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/snap/4.snap rename to crates/meili-snap/src/snapshots/lib.rs/snap/4.snap diff --git a/meili-snap/src/snapshots/lib.rs/snap/5.snap b/crates/meili-snap/src/snapshots/lib.rs/snap/5.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/snap/5.snap rename to crates/meili-snap/src/snapshots/lib.rs/snap/5.snap diff --git a/meili-snap/src/snapshots/lib.rs/snap/6.snap b/crates/meili-snap/src/snapshots/lib.rs/snap/6.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/snap/6.snap rename to crates/meili-snap/src/snapshots/lib.rs/snap/6.snap diff --git a/meili-snap/src/snapshots/lib.rs/snap/7.snap b/crates/meili-snap/src/snapshots/lib.rs/snap/7.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/snap/7.snap rename to crates/meili-snap/src/snapshots/lib.rs/snap/7.snap diff --git a/meili-snap/src/snapshots/lib.rs/snap/snap_name_1.snap b/crates/meili-snap/src/snapshots/lib.rs/snap/snap_name_1.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/snap/snap_name_1.snap rename to crates/meili-snap/src/snapshots/lib.rs/snap/snap_name_1.snap diff --git a/meili-snap/src/snapshots/lib.rs/some_test/4.snap b/crates/meili-snap/src/snapshots/lib.rs/some_test/4.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/some_test/4.snap rename to crates/meili-snap/src/snapshots/lib.rs/some_test/4.snap diff --git a/meili-snap/src/snapshots/lib.rs/some_test/5.snap b/crates/meili-snap/src/snapshots/lib.rs/some_test/5.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/some_test/5.snap rename to crates/meili-snap/src/snapshots/lib.rs/some_test/5.snap diff --git a/meili-snap/src/snapshots/lib.rs/some_test/6.snap b/crates/meili-snap/src/snapshots/lib.rs/some_test/6.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/some_test/6.snap rename to crates/meili-snap/src/snapshots/lib.rs/some_test/6.snap diff --git a/meili-snap/src/snapshots/lib.rs/some_test/7.snap b/crates/meili-snap/src/snapshots/lib.rs/some_test/7.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/some_test/7.snap rename to crates/meili-snap/src/snapshots/lib.rs/some_test/7.snap diff --git a/meili-snap/src/snapshots/lib.rs/some_test/snap_name_1.snap b/crates/meili-snap/src/snapshots/lib.rs/some_test/snap_name_1.snap similarity index 100% rename from meili-snap/src/snapshots/lib.rs/some_test/snap_name_1.snap rename to crates/meili-snap/src/snapshots/lib.rs/some_test/snap_name_1.snap diff --git a/meilisearch-auth/Cargo.toml b/crates/meilisearch-auth/Cargo.toml similarity index 100% rename from meilisearch-auth/Cargo.toml rename to crates/meilisearch-auth/Cargo.toml diff --git a/meilisearch-auth/src/dump.rs b/crates/meilisearch-auth/src/dump.rs similarity index 100% rename from meilisearch-auth/src/dump.rs rename to crates/meilisearch-auth/src/dump.rs diff --git a/meilisearch-auth/src/error.rs b/crates/meilisearch-auth/src/error.rs similarity index 100% rename from meilisearch-auth/src/error.rs rename to crates/meilisearch-auth/src/error.rs diff --git a/meilisearch-auth/src/lib.rs b/crates/meilisearch-auth/src/lib.rs similarity index 100% rename from meilisearch-auth/src/lib.rs rename to crates/meilisearch-auth/src/lib.rs diff --git a/meilisearch-auth/src/store.rs b/crates/meilisearch-auth/src/store.rs similarity index 100% rename from meilisearch-auth/src/store.rs rename to crates/meilisearch-auth/src/store.rs diff --git a/meilisearch-types/Cargo.toml b/crates/meilisearch-types/Cargo.toml similarity index 95% rename from meilisearch-types/Cargo.toml rename to crates/meilisearch-types/Cargo.toml index cb4937e57..0dae024f2 100644 --- a/meilisearch-types/Cargo.toml +++ b/crates/meilisearch-types/Cargo.toml @@ -66,5 +66,8 @@ khmer = ["milli/khmer"] vietnamese = ["milli/vietnamese"] # force swedish character recomposition swedish-recomposition = ["milli/swedish-recomposition"] -# force german character recomposition +# allow german tokenization german = ["milli/german"] +# allow turkish normalization +turkish = ["milli/turkish"] + diff --git a/meilisearch-types/src/compression.rs b/crates/meilisearch-types/src/compression.rs similarity index 100% rename from meilisearch-types/src/compression.rs rename to crates/meilisearch-types/src/compression.rs diff --git a/meilisearch-types/src/deserr/mod.rs b/crates/meilisearch-types/src/deserr/mod.rs similarity index 100% rename from meilisearch-types/src/deserr/mod.rs rename to crates/meilisearch-types/src/deserr/mod.rs diff --git a/meilisearch-types/src/deserr/query_params.rs b/crates/meilisearch-types/src/deserr/query_params.rs similarity index 100% rename from meilisearch-types/src/deserr/query_params.rs rename to crates/meilisearch-types/src/deserr/query_params.rs diff --git a/meilisearch-types/src/document_formats.rs b/crates/meilisearch-types/src/document_formats.rs similarity index 100% rename from meilisearch-types/src/document_formats.rs rename to crates/meilisearch-types/src/document_formats.rs diff --git a/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs similarity index 99% rename from meilisearch-types/src/error.rs rename to crates/meilisearch-types/src/error.rs index f755998a1..514ed18c3 100644 --- a/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -543,7 +543,8 @@ impl fmt::Display for deserr_codes::InvalidSimilarId { f, "the value of `id` is invalid. \ A document identifier can be of type integer or string, \ - only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)." + only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_), \ + and can not be more than 512 bytes." ) } } diff --git a/meilisearch-types/src/facet_values_sort.rs b/crates/meilisearch-types/src/facet_values_sort.rs similarity index 100% rename from meilisearch-types/src/facet_values_sort.rs rename to crates/meilisearch-types/src/facet_values_sort.rs diff --git a/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs similarity index 100% rename from meilisearch-types/src/features.rs rename to crates/meilisearch-types/src/features.rs diff --git a/meilisearch-types/src/index_uid.rs b/crates/meilisearch-types/src/index_uid.rs similarity index 95% rename from meilisearch-types/src/index_uid.rs rename to crates/meilisearch-types/src/index_uid.rs index d64a6658d..03a31a82f 100644 --- a/meilisearch-types/src/index_uid.rs +++ b/crates/meilisearch-types/src/index_uid.rs @@ -88,7 +88,8 @@ impl fmt::Display for IndexUidFormatError { f, "`{}` is not a valid index uid. Index uid can be an \ integer or a string containing only alphanumeric \ - characters, hyphens (-) and underscores (_).", + characters, hyphens (-) and underscores (_), \ + and can not be more than 512 bytes.", self.invalid_uid, ) } diff --git a/meilisearch-types/src/index_uid_pattern.rs b/crates/meilisearch-types/src/index_uid_pattern.rs similarity index 100% rename from meilisearch-types/src/index_uid_pattern.rs rename to crates/meilisearch-types/src/index_uid_pattern.rs diff --git a/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs similarity index 100% rename from meilisearch-types/src/keys.rs rename to crates/meilisearch-types/src/keys.rs diff --git a/meilisearch-types/src/lib.rs b/crates/meilisearch-types/src/lib.rs similarity index 100% rename from meilisearch-types/src/lib.rs rename to crates/meilisearch-types/src/lib.rs diff --git a/meilisearch-types/src/locales.rs b/crates/meilisearch-types/src/locales.rs similarity index 100% rename from meilisearch-types/src/locales.rs rename to crates/meilisearch-types/src/locales.rs diff --git a/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs similarity index 100% rename from meilisearch-types/src/settings.rs rename to crates/meilisearch-types/src/settings.rs diff --git a/meilisearch-types/src/star_or.rs b/crates/meilisearch-types/src/star_or.rs similarity index 100% rename from meilisearch-types/src/star_or.rs rename to crates/meilisearch-types/src/star_or.rs diff --git a/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs similarity index 100% rename from meilisearch-types/src/task_view.rs rename to crates/meilisearch-types/src/task_view.rs diff --git a/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs similarity index 100% rename from meilisearch-types/src/tasks.rs rename to crates/meilisearch-types/src/tasks.rs diff --git a/meilisearch-types/src/versioning.rs b/crates/meilisearch-types/src/versioning.rs similarity index 100% rename from meilisearch-types/src/versioning.rs rename to crates/meilisearch-types/src/versioning.rs diff --git a/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml similarity index 94% rename from meilisearch/Cargo.toml rename to crates/meilisearch/Cargo.toml index 177ab6ee2..b11d90151 100644 --- a/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -75,7 +75,7 @@ reqwest = { version = "0.12.5", features = [ rustls = { version = "0.23.11", features = ["ring"], default-features = false } rustls-pki-types = { version = "1.7.0", features = ["alloc"] } rustls-pemfile = "2.1.2" -segment = { version = "0.2.4", optional = true } +segment = { version = "0.2.4" } serde = { version = "1.0.204", features = ["derive"] } serde_json = { version = "1.0.120", features = ["preserve_order"] } sha2 = "0.10.8" @@ -104,6 +104,7 @@ tracing-trace = { version = "0.1.0", path = "../tracing-trace" } tracing-actix-web = "0.7.11" build-info = { version = "1.7.0", path = "../build-info" } roaring = "0.10.2" +mopa-maintained = "0.2.3" [dev-dependencies] actix-rt = "2.10.0" @@ -131,8 +132,7 @@ tempfile = { version = "3.10.1", optional = true } zip = { version = "2.1.3", optional = true } [features] -default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"] -analytics = ["segment"] +default = ["meilisearch-types/all-tokenizations", "mini-dashboard"] mini-dashboard = [ "static-files", "anyhow", @@ -154,7 +154,8 @@ khmer = ["meilisearch-types/khmer"] vietnamese = ["meilisearch-types/vietnamese"] swedish-recomposition = ["meilisearch-types/swedish-recomposition"] german = ["meilisearch-types/german"] +turkish = ["meilisearch-types/turkish"] [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.14/build.zip" -sha1 = "592d1b5a3459d621d0aae1dded8fe3154f5c38fe" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.15/build.zip" +sha1 = "d057600b4a839a2e0c0be7a372cd1b2683f3ca7e" diff --git a/meilisearch/build.rs b/crates/meilisearch/build.rs similarity index 100% rename from meilisearch/build.rs rename to crates/meilisearch/build.rs diff --git a/meilisearch/src/analytics/mock_analytics.rs b/crates/meilisearch/src/analytics/mock_analytics.rs similarity index 100% rename from meilisearch/src/analytics/mock_analytics.rs rename to crates/meilisearch/src/analytics/mock_analytics.rs diff --git a/crates/meilisearch/src/analytics/mod.rs b/crates/meilisearch/src/analytics/mod.rs new file mode 100644 index 000000000..bd14b0bfa --- /dev/null +++ b/crates/meilisearch/src/analytics/mod.rs @@ -0,0 +1,166 @@ +pub mod segment_analytics; + +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::Arc; + +use actix_web::HttpRequest; +use index_scheduler::IndexScheduler; +use meilisearch_auth::AuthController; +use meilisearch_types::InstanceUid; +use mopa::mopafy; +use once_cell::sync::Lazy; +use platform_dirs::AppDirs; + +// if the feature analytics is enabled we use the real analytics +pub type SegmentAnalytics = segment_analytics::SegmentAnalytics; + +use crate::Opt; + +/// A macro used to quickly define events that don't aggregate or send anything besides an empty event with its name. +#[macro_export] +macro_rules! empty_analytics { + ($struct_name:ident, $event_name:literal) => { + #[derive(Default)] + struct $struct_name {} + + impl $crate::analytics::Aggregate for $struct_name { + fn event_name(&self) -> &'static str { + $event_name + } + + fn aggregate(self: Box, _other: Box) -> Box { + self + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::json!({}) + } + } + }; +} + +/// The Meilisearch config dir: +/// `~/.config/Meilisearch` on *NIX or *BSD. +/// `~/Library/ApplicationSupport` on macOS. +/// `%APPDATA` (= `C:\Users%USERNAME%\AppData\Roaming`) on windows. +static MEILISEARCH_CONFIG_PATH: Lazy> = + Lazy::new(|| AppDirs::new(Some("Meilisearch"), false).map(|appdir| appdir.config_dir)); + +fn config_user_id_path(db_path: &Path) -> Option { + db_path + .canonicalize() + .ok() + .map(|path| path.join("instance-uid").display().to_string().replace('/', "-")) + .zip(MEILISEARCH_CONFIG_PATH.as_ref()) + .map(|(filename, config_path)| config_path.join(filename.trim_start_matches('-'))) +} + +/// Look for the instance-uid in the `data.ms` or in `~/.config/Meilisearch/path-to-db-instance-uid` +fn find_user_id(db_path: &Path) -> Option { + fs::read_to_string(db_path.join("instance-uid")) + .ok() + .or_else(|| fs::read_to_string(config_user_id_path(db_path)?).ok()) + .and_then(|uid| InstanceUid::from_str(&uid).ok()) +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum DocumentDeletionKind { + PerDocumentId, + ClearAll, + PerBatch, + PerFilter, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum DocumentFetchKind { + PerDocumentId { retrieve_vectors: bool }, + Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool }, +} + +/// To send an event to segment, your event must be able to aggregate itself with another event of the same type. +pub trait Aggregate: 'static + mopa::Any + Send { + /// The name of the event that will be sent to segment. + fn event_name(&self) -> &'static str; + + /// Will be called every time an event has been used twice before segment flushed its buffer. + fn aggregate(self: Box, new: Box) -> Box + where + Self: Sized; + + /// Converts your structure to the final event that'll be sent to segment. + fn into_event(self: Box) -> serde_json::Value; +} + +mopafy!(Aggregate); + +/// Helper trait to define multiple aggregates with the same content but a different name. +/// Commonly used when you must aggregate a search with POST or with GET, for example. +pub trait AggregateMethod: 'static + Default + Send { + fn event_name() -> &'static str; +} + +/// A macro used to quickly define multiple aggregate method with their name +/// Usage: +/// ```rust +/// use meilisearch::aggregate_methods; +/// +/// aggregate_methods!( +/// SearchGET => "Documents Searched GET", +/// SearchPOST => "Documents Searched POST", +/// ); +/// ``` +#[macro_export] +macro_rules! aggregate_methods { + ($method:ident => $event_name:literal) => { + #[derive(Default)] + pub struct $method {} + + impl $crate::analytics::AggregateMethod for $method { + fn event_name() -> &'static str { + $event_name + } + } + }; + ($($method:ident => $event_name:literal,)+) => { + $( + aggregate_methods!($method => $event_name); + )+ + + }; +} + +#[derive(Clone)] +pub struct Analytics { + segment: Option>, +} + +impl Analytics { + pub async fn new( + opt: &Opt, + index_scheduler: Arc, + auth_controller: Arc, + ) -> Self { + if opt.no_analytics { + Self { segment: None } + } else { + Self { segment: SegmentAnalytics::new(opt, index_scheduler, auth_controller).await } + } + } + + pub fn no_analytics() -> Self { + Self { segment: None } + } + + pub fn instance_uid(&self) -> Option<&InstanceUid> { + self.segment.as_ref().map(|segment| segment.instance_uid.as_ref()) + } + + /// The method used to publish most analytics that do not need to be batched every hours + pub fn publish(&self, event: T, request: &HttpRequest) { + if let Some(ref segment) = self.segment { + let _ = segment.sender.try_send(segment_analytics::Message::new(event, request)); + } + } +} diff --git a/crates/meilisearch/src/analytics/segment_analytics.rs b/crates/meilisearch/src/analytics/segment_analytics.rs new file mode 100644 index 000000000..7dc746b14 --- /dev/null +++ b/crates/meilisearch/src/analytics/segment_analytics.rs @@ -0,0 +1,484 @@ +use std::any::TypeId; +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use actix_web::http::header::USER_AGENT; +use actix_web::HttpRequest; +use byte_unit::Byte; +use index_scheduler::IndexScheduler; +use meilisearch_auth::{AuthController, AuthFilter}; +use meilisearch_types::features::RuntimeTogglableFeatures; +use meilisearch_types::InstanceUid; +use once_cell::sync::Lazy; +use segment::message::{Identify, Track, User}; +use segment::{AutoBatcher, Batcher, HttpClient}; +use serde::Serialize; +use serde_json::{json, Value}; +use sysinfo::{Disks, System}; +use time::OffsetDateTime; +use tokio::select; +use tokio::sync::mpsc::{self, Receiver, Sender}; +use uuid::Uuid; + +use super::{config_user_id_path, Aggregate, MEILISEARCH_CONFIG_PATH}; +use crate::option::{ + default_http_addr, IndexerOpts, LogMode, MaxMemory, MaxThreads, ScheduleSnapshot, +}; +use crate::routes::{create_all_stats, Stats}; +use crate::Opt; + +const ANALYTICS_HEADER: &str = "X-Meilisearch-Client"; + +/// Write the instance-uid in the `data.ms` and in `~/.config/MeiliSearch/path-to-db-instance-uid`. Ignore the errors. +fn write_user_id(db_path: &Path, user_id: &InstanceUid) { + let _ = fs::write(db_path.join("instance-uid"), user_id.to_string()); + if let Some((meilisearch_config_path, user_id_path)) = + MEILISEARCH_CONFIG_PATH.as_ref().zip(config_user_id_path(db_path)) + { + let _ = fs::create_dir_all(meilisearch_config_path); + let _ = fs::write(user_id_path, user_id.to_string()); + } +} + +const SEGMENT_API_KEY: &str = "P3FWhhEsJiEDCuEHpmcN9DHcK4hVfBvb"; + +pub fn extract_user_agents(request: &HttpRequest) -> HashSet { + request + .headers() + .get(ANALYTICS_HEADER) + .or_else(|| request.headers().get(USER_AGENT)) + .and_then(|header| header.to_str().ok()) + .unwrap_or("unknown") + .split(';') + .map(str::trim) + .map(ToString::to_string) + .collect() +} + +pub struct Message { + // Since the type_id is solved statically we cannot retrieve it from the Box. + // Thus we have to send it in the message directly. + type_id: TypeId, + // Same for the aggregate function. + #[allow(clippy::type_complexity)] + aggregator_function: fn(Box, Box) -> Option>, + event: Event, +} + +pub struct Event { + original: Box, + timestamp: OffsetDateTime, + user_agents: HashSet, + total: usize, +} + +/// This function should always be called on the same type. If `this` and `other` +/// aren't the same type the function will do nothing and return `None`. +fn downcast_aggregate( + old: Box, + new: Box, +) -> Option> { + if old.is::() && new.is::() { + // Both the two following lines cannot fail, but just to be sure we don't crash, we're still avoiding unwrapping + let this = old.downcast::().ok()?; + let other = new.downcast::().ok()?; + Some(ConcreteType::aggregate(this, other)) + } else { + None + } +} + +impl Message { + pub fn new(event: T, request: &HttpRequest) -> Self { + Self { + type_id: TypeId::of::(), + event: Event { + original: Box::new(event), + timestamp: OffsetDateTime::now_utc(), + user_agents: extract_user_agents(request), + total: 1, + }, + aggregator_function: downcast_aggregate::, + } + } +} + +pub struct SegmentAnalytics { + pub instance_uid: InstanceUid, + pub user: User, + pub sender: Sender, +} + +impl SegmentAnalytics { + #[allow(clippy::new_ret_no_self)] + pub async fn new( + opt: &Opt, + index_scheduler: Arc, + auth_controller: Arc, + ) -> Option> { + let instance_uid = super::find_user_id(&opt.db_path); + let first_time_run = instance_uid.is_none(); + let instance_uid = instance_uid.unwrap_or_else(Uuid::new_v4); + write_user_id(&opt.db_path, &instance_uid); + + let client = reqwest::Client::builder().connect_timeout(Duration::from_secs(10)).build(); + + // if reqwest throws an error we won't be able to send analytics + if client.is_err() { + return None; + } + + let client = + HttpClient::new(client.unwrap(), "https://telemetry.meilisearch.com".to_string()); + let user = User::UserId { user_id: instance_uid.to_string() }; + let mut batcher = AutoBatcher::new(client, Batcher::new(None), SEGMENT_API_KEY.to_string()); + + // If Meilisearch is Launched for the first time: + // 1. Send an event Launched associated to the user `total_launch`. + // 2. Batch an event Launched with the real instance-id and send it in one hour. + if first_time_run { + let _ = batcher + .push(Track { + user: User::UserId { user_id: "total_launch".to_string() }, + event: "Launched".to_string(), + ..Default::default() + }) + .await; + let _ = batcher.flush().await; + let _ = batcher + .push(Track { + user: user.clone(), + event: "Launched".to_string(), + ..Default::default() + }) + .await; + } + + let (sender, inbox) = mpsc::channel(100); // How many analytics can we bufferize + + let segment = Box::new(Segment { + inbox, + user: user.clone(), + opt: opt.clone(), + batcher, + events: HashMap::new(), + }); + tokio::spawn(segment.run(index_scheduler.clone(), auth_controller.clone())); + + let this = Self { instance_uid, sender, user: user.clone() }; + + Some(Arc::new(this)) + } +} + +/// This structure represent the `infos` field we send in the analytics. +/// It's quite close to the `Opt` structure except all sensitive informations +/// have been simplified to a boolean. +/// It's send as-is in amplitude thus you should never update a name of the +/// struct without the approval of the PM. +#[derive(Debug, Clone, Serialize)] +struct Infos { + env: String, + experimental_contains_filter: bool, + experimental_vector_store: bool, + experimental_enable_metrics: bool, + experimental_edit_documents_by_function: bool, + experimental_search_queue_size: usize, + experimental_drop_search_after: usize, + experimental_nb_searches_per_core: usize, + experimental_logs_mode: LogMode, + experimental_replication_parameters: bool, + experimental_enable_logs_route: bool, + experimental_reduce_indexing_memory_usage: bool, + experimental_max_number_of_batched_tasks: usize, + gpu_enabled: bool, + db_path: bool, + import_dump: bool, + dump_dir: bool, + ignore_missing_dump: bool, + ignore_dump_if_db_exists: bool, + import_snapshot: bool, + schedule_snapshot: Option, + snapshot_dir: bool, + ignore_missing_snapshot: bool, + ignore_snapshot_if_db_exists: bool, + http_addr: bool, + http_payload_size_limit: Byte, + task_queue_webhook: bool, + task_webhook_authorization_header: bool, + log_level: String, + max_indexing_memory: MaxMemory, + max_indexing_threads: MaxThreads, + with_configuration_file: bool, + ssl_auth_path: bool, + ssl_cert_path: bool, + ssl_key_path: bool, + ssl_ocsp_path: bool, + ssl_require_auth: bool, + ssl_resumption: bool, + ssl_tickets: bool, +} + +impl Infos { + pub fn new(options: Opt, features: RuntimeTogglableFeatures) -> Self { + // We wants to decompose this whole struct by hand to be sure we don't forget + // to add analytics when we add a field in the Opt. + // Thus we must not insert `..` at the end. + let Opt { + db_path, + experimental_contains_filter, + experimental_enable_metrics, + experimental_search_queue_size, + experimental_drop_search_after, + experimental_nb_searches_per_core, + experimental_logs_mode, + experimental_replication_parameters, + experimental_enable_logs_route, + experimental_reduce_indexing_memory_usage, + experimental_max_number_of_batched_tasks, + http_addr, + master_key: _, + env, + task_webhook_url, + task_webhook_authorization_header, + max_index_size: _, + max_task_db_size: _, + http_payload_size_limit, + ssl_cert_path, + ssl_key_path, + ssl_auth_path, + ssl_ocsp_path, + ssl_require_auth, + ssl_resumption, + ssl_tickets, + import_snapshot, + ignore_missing_snapshot, + ignore_snapshot_if_db_exists, + snapshot_dir, + schedule_snapshot, + import_dump, + ignore_missing_dump, + ignore_dump_if_db_exists, + dump_dir, + log_level, + indexer_options, + config_file_path, + no_analytics: _, + } = options; + + let schedule_snapshot = match schedule_snapshot { + ScheduleSnapshot::Disabled => None, + ScheduleSnapshot::Enabled(interval) => Some(interval), + }; + + let IndexerOpts { max_indexing_memory, max_indexing_threads, skip_index_budget: _ } = + indexer_options; + + let RuntimeTogglableFeatures { + vector_store, + metrics, + logs_route, + edit_documents_by_function, + contains_filter, + } = features; + + // We're going to override every sensible information. + // We consider information sensible if it contains a path, an address, or a key. + Self { + env, + experimental_contains_filter: experimental_contains_filter | contains_filter, + experimental_vector_store: vector_store, + experimental_edit_documents_by_function: edit_documents_by_function, + experimental_enable_metrics: experimental_enable_metrics | metrics, + experimental_search_queue_size, + experimental_drop_search_after: experimental_drop_search_after.into(), + experimental_nb_searches_per_core: experimental_nb_searches_per_core.into(), + experimental_logs_mode, + experimental_replication_parameters, + experimental_enable_logs_route: experimental_enable_logs_route | logs_route, + experimental_reduce_indexing_memory_usage, + gpu_enabled: meilisearch_types::milli::vector::is_cuda_enabled(), + db_path: db_path != PathBuf::from("./data.ms"), + import_dump: import_dump.is_some(), + dump_dir: dump_dir != PathBuf::from("dumps/"), + ignore_missing_dump, + ignore_dump_if_db_exists, + import_snapshot: import_snapshot.is_some(), + schedule_snapshot, + snapshot_dir: snapshot_dir != PathBuf::from("snapshots/"), + ignore_missing_snapshot, + ignore_snapshot_if_db_exists, + http_addr: http_addr != default_http_addr(), + http_payload_size_limit, + experimental_max_number_of_batched_tasks, + task_queue_webhook: task_webhook_url.is_some(), + task_webhook_authorization_header: task_webhook_authorization_header.is_some(), + log_level: log_level.to_string(), + max_indexing_memory, + max_indexing_threads, + with_configuration_file: config_file_path.is_some(), + ssl_auth_path: ssl_auth_path.is_some(), + ssl_cert_path: ssl_cert_path.is_some(), + ssl_key_path: ssl_key_path.is_some(), + ssl_ocsp_path: ssl_ocsp_path.is_some(), + ssl_require_auth, + ssl_resumption, + ssl_tickets, + } + } +} + +pub struct Segment { + inbox: Receiver, + user: User, + opt: Opt, + batcher: AutoBatcher, + events: HashMap, +} + +impl Segment { + fn compute_traits(opt: &Opt, stats: Stats, features: RuntimeTogglableFeatures) -> Value { + static FIRST_START_TIMESTAMP: Lazy = Lazy::new(Instant::now); + static SYSTEM: Lazy = Lazy::new(|| { + let disks = Disks::new_with_refreshed_list(); + let mut sys = System::new_all(); + sys.refresh_all(); + let kernel_version = System::kernel_version() + .and_then(|k| k.split_once('-').map(|(k, _)| k.to_string())); + json!({ + "distribution": System::name(), + "kernel_version": kernel_version, + "cores": sys.cpus().len(), + "ram_size": sys.total_memory(), + "disk_size": disks.iter().map(|disk| disk.total_space()).max(), + "server_provider": std::env::var("MEILI_SERVER_PROVIDER").ok(), + }) + }); + let number_of_documents = + stats.indexes.values().map(|index| index.number_of_documents).collect::>(); + + json!({ + "start_since_days": FIRST_START_TIMESTAMP.elapsed().as_secs() / (60 * 60 * 24), // one day + "system": *SYSTEM, + "stats": { + "database_size": stats.database_size, + "indexes_number": stats.indexes.len(), + "documents_number": number_of_documents, + }, + "infos": Infos::new(opt.clone(), features), + }) + } + + async fn run( + mut self, + index_scheduler: Arc, + auth_controller: Arc, + ) { + const INTERVAL: Duration = Duration::from_secs(60 * 60); // one hour + // The first batch must be sent after one hour. + let mut interval = + tokio::time::interval_at(tokio::time::Instant::now() + INTERVAL, INTERVAL); + + loop { + select! { + _ = interval.tick() => { + self.tick(index_scheduler.clone(), auth_controller.clone()).await; + }, + Some(msg) = self.inbox.recv() => { + self.handle_msg(msg); + } + } + } + } + + fn handle_msg(&mut self, Message { type_id, aggregator_function, event }: Message) { + let new_event = match self.events.remove(&type_id) { + Some(old) => { + // The function should never fail since we retrieved the corresponding TypeId in the map. But in the unfortunate + // case it could happens we're going to silently ignore the error + let Some(original) = (aggregator_function)(old.original, event.original) else { + return; + }; + Event { + original, + // We always want to return the FIRST timestamp ever encountered + timestamp: old.timestamp, + user_agents: old.user_agents.union(&event.user_agents).cloned().collect(), + total: old.total.saturating_add(event.total), + } + } + None => event, + }; + self.events.insert(type_id, new_event); + } + + async fn tick( + &mut self, + index_scheduler: Arc, + auth_controller: Arc, + ) { + if let Ok(stats) = create_all_stats( + index_scheduler.clone().into(), + auth_controller.into(), + &AuthFilter::default(), + ) { + // Replace the version number with the prototype name if any. + let version = if let Some(prototype) = build_info::DescribeResult::from_build() + .and_then(|describe| describe.as_prototype()) + { + prototype + } else { + env!("CARGO_PKG_VERSION") + }; + + let _ = self + .batcher + .push(Identify { + context: Some(json!({ + "app": { + "version": version.to_string(), + }, + })), + user: self.user.clone(), + traits: Self::compute_traits( + &self.opt, + stats, + index_scheduler.features().runtime_features(), + ), + ..Default::default() + }) + .await; + } + + // We empty the list of events + let events = std::mem::take(&mut self.events); + + for (_, event) in events { + let Event { original, timestamp, user_agents, total } = event; + let name = original.event_name(); + let mut properties = original.into_event(); + if properties["user-agent"].is_null() { + properties["user-agent"] = json!(user_agents); + }; + if properties["requests"]["total_received"].is_null() { + properties["requests"]["total_received"] = total.into(); + }; + + let _ = self + .batcher + .push(Track { + user: self.user.clone(), + event: name.to_string(), + properties, + timestamp: Some(timestamp), + ..Default::default() + }) + .await; + } + + let _ = self.batcher.flush().await; + } +} diff --git a/meilisearch/src/error.rs b/crates/meilisearch/src/error.rs similarity index 100% rename from meilisearch/src/error.rs rename to crates/meilisearch/src/error.rs diff --git a/meilisearch/src/extractors/authentication/error.rs b/crates/meilisearch/src/extractors/authentication/error.rs similarity index 100% rename from meilisearch/src/extractors/authentication/error.rs rename to crates/meilisearch/src/extractors/authentication/error.rs diff --git a/meilisearch/src/extractors/authentication/mod.rs b/crates/meilisearch/src/extractors/authentication/mod.rs similarity index 100% rename from meilisearch/src/extractors/authentication/mod.rs rename to crates/meilisearch/src/extractors/authentication/mod.rs diff --git a/meilisearch/src/extractors/mod.rs b/crates/meilisearch/src/extractors/mod.rs similarity index 100% rename from meilisearch/src/extractors/mod.rs rename to crates/meilisearch/src/extractors/mod.rs diff --git a/meilisearch/src/extractors/payload.rs b/crates/meilisearch/src/extractors/payload.rs similarity index 100% rename from meilisearch/src/extractors/payload.rs rename to crates/meilisearch/src/extractors/payload.rs diff --git a/meilisearch/src/extractors/sequential_extractor.rs b/crates/meilisearch/src/extractors/sequential_extractor.rs similarity index 100% rename from meilisearch/src/extractors/sequential_extractor.rs rename to crates/meilisearch/src/extractors/sequential_extractor.rs diff --git a/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs similarity index 99% rename from meilisearch/src/lib.rs rename to crates/meilisearch/src/lib.rs index b24f18fae..633ad2776 100644 --- a/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -120,7 +120,7 @@ pub fn create_app( search_queue: Data, opt: Opt, logs: (LogRouteHandle, LogStderrHandle), - analytics: Arc, + analytics: Data, enable_dashboard: bool, ) -> actix_web::App< impl ServiceFactory< @@ -473,14 +473,14 @@ pub fn configure_data( search_queue: Data, opt: &Opt, (logs_route, logs_stderr): (LogRouteHandle, LogStderrHandle), - analytics: Arc, + analytics: Data, ) { let http_payload_size_limit = opt.http_payload_size_limit.as_u64() as usize; config .app_data(index_scheduler) .app_data(auth) .app_data(search_queue) - .app_data(web::Data::from(analytics)) + .app_data(analytics) .app_data(web::Data::new(logs_route)) .app_data(web::Data::new(logs_stderr)) .app_data(web::Data::new(opt.clone())) diff --git a/meilisearch/src/main.rs b/crates/meilisearch/src/main.rs similarity index 94% rename from meilisearch/src/main.rs rename to crates/meilisearch/src/main.rs index b66bfc5b8..c0652bf1e 100644 --- a/meilisearch/src/main.rs +++ b/crates/meilisearch/src/main.rs @@ -5,6 +5,7 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use std::thread::available_parallelism; +use std::time::Duration; use actix_web::http::KeepAlive; use actix_web::web::Data; @@ -123,19 +124,12 @@ async fn try_main() -> anyhow::Result<()> { let (index_scheduler, auth_controller) = setup_meilisearch(&opt)?; - #[cfg(all(not(debug_assertions), feature = "analytics"))] - let analytics = if !opt.no_analytics { - analytics::SegmentAnalytics::new(&opt, index_scheduler.clone(), auth_controller.clone()) - .await - } else { - analytics::MockAnalytics::new(&opt) - }; - #[cfg(any(debug_assertions, not(feature = "analytics")))] - let analytics = analytics::MockAnalytics::new(&opt); + let analytics = + analytics::Analytics::new(&opt, index_scheduler.clone(), auth_controller.clone()).await; print_launch_resume(&opt, analytics.clone(), config_read_from); - run_http(index_scheduler, auth_controller, opt, log_handle, analytics).await?; + run_http(index_scheduler, auth_controller, opt, log_handle, Arc::new(analytics)).await?; Ok(()) } @@ -145,16 +139,23 @@ async fn run_http( auth_controller: Arc, opt: Opt, logs: (LogRouteHandle, LogStderrHandle), - analytics: Arc, + analytics: Arc, ) -> anyhow::Result<()> { let enable_dashboard = &opt.env == "development"; let opt_clone = opt.clone(); let index_scheduler = Data::from(index_scheduler); let auth_controller = Data::from(auth_controller); + let analytics = Data::from(analytics); let search_queue = SearchQueue::new( opt.experimental_search_queue_size, - available_parallelism().unwrap_or(NonZeroUsize::new(2).unwrap()), - ); + available_parallelism() + .unwrap_or(NonZeroUsize::new(2).unwrap()) + .checked_mul(opt.experimental_nb_searches_per_core) + .unwrap_or(NonZeroUsize::MAX), + ) + .with_time_to_abort(Duration::from_secs( + usize::from(opt.experimental_drop_search_after) as u64 + )); let search_queue = Data::new(search_queue); let http_server = HttpServer::new(move || { @@ -180,11 +181,7 @@ async fn run_http( Ok(()) } -pub fn print_launch_resume( - opt: &Opt, - analytics: Arc, - config_read_from: Option, -) { +pub fn print_launch_resume(opt: &Opt, analytics: Analytics, config_read_from: Option) { let build_info = build_info::BuildInfo::from_build(); let protocol = @@ -226,7 +223,6 @@ pub fn print_launch_resume( eprintln!("Prototype:\t\t{:?}", prototype); } - #[cfg(all(not(debug_assertions), feature = "analytics"))] { if !opt.no_analytics { eprintln!( diff --git a/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs similarity index 100% rename from meilisearch/src/metrics.rs rename to crates/meilisearch/src/metrics.rs diff --git a/meilisearch/src/middleware.rs b/crates/meilisearch/src/middleware.rs similarity index 100% rename from meilisearch/src/middleware.rs rename to crates/meilisearch/src/middleware.rs diff --git a/meilisearch/src/option.rs b/crates/meilisearch/src/option.rs similarity index 94% rename from meilisearch/src/option.rs rename to crates/meilisearch/src/option.rs index 3799bdcb7..7e87a5a2c 100644 --- a/meilisearch/src/option.rs +++ b/crates/meilisearch/src/option.rs @@ -2,7 +2,7 @@ use std::env::VarError; use std::ffi::OsStr; use std::fmt::Display; use std::io::{BufReader, Read}; -use std::num::ParseIntError; +use std::num::{NonZeroUsize, ParseIntError}; use std::ops::Deref; use std::path::PathBuf; use std::str::FromStr; @@ -29,7 +29,6 @@ const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY"; const MEILI_ENV: &str = "MEILI_ENV"; const MEILI_TASK_WEBHOOK_URL: &str = "MEILI_TASK_WEBHOOK_URL"; const MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER: &str = "MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER"; -#[cfg(feature = "analytics")] const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS"; const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT"; const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH"; @@ -55,6 +54,8 @@ const MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE: &str = "MEILI_EXPERIMENTAL_ENABLE_LO const MEILI_EXPERIMENTAL_CONTAINS_FILTER: &str = "MEILI_EXPERIMENTAL_CONTAINS_FILTER"; const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS"; const MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE: &str = "MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE"; +const MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER: &str = "MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER"; +const MEILI_EXPERIMENTAL_NB_SEARCHES_PER_CORE: &str = "MEILI_EXPERIMENTAL_NB_SEARCHES_PER_CORE"; const MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: &str = "MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE"; const MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS: &str = @@ -208,7 +209,6 @@ pub struct Opt { /// Meilisearch automatically collects data from all instances that do not opt out using this flag. /// All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted /// at any time. - #[cfg(feature = "analytics")] #[serde(default)] // we can't send true #[clap(long, env = MEILI_NO_ANALYTICS)] pub no_analytics: bool, @@ -357,10 +357,26 @@ pub struct Opt { /// Lets you customize the size of the search queue. Meilisearch processes your search requests as fast as possible but once the /// queue is full it starts returning HTTP 503, Service Unavailable. /// The default value is 1000. - #[clap(long, env = MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE, default_value_t = 1000)] - #[serde(default)] + #[clap(long, env = MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE, default_value_t = default_experimental_search_queue_size())] + #[serde(default = "default_experimental_search_queue_size")] pub experimental_search_queue_size: usize, + /// Experimental drop search after. For more information, see: + /// + /// Let you customize after how many seconds Meilisearch should consider a search request irrelevant and drop it. + /// The default value is 60. + #[clap(long, env = MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER, default_value_t = default_drop_search_after())] + #[serde(default = "default_drop_search_after")] + pub experimental_drop_search_after: NonZeroUsize, + + /// Experimental number of searches per core. For more information, see: + /// + /// Lets you customize how many search requests can run on each core concurrently. + /// The default value is 4. + #[clap(long, env = MEILI_EXPERIMENTAL_NB_SEARCHES_PER_CORE, default_value_t = default_nb_searches_per_core())] + #[serde(default = "default_nb_searches_per_core")] + pub experimental_nb_searches_per_core: NonZeroUsize, + /// Experimental logs mode feature. For more information, see: /// /// Change the mode of the logs on the console. @@ -407,7 +423,6 @@ pub struct Opt { impl Opt { /// Whether analytics should be enabled or not. - #[cfg(all(not(debug_assertions), feature = "analytics"))] pub fn analytics(&self) -> bool { !self.no_analytics } @@ -487,11 +502,12 @@ impl Opt { ignore_missing_dump: _, ignore_dump_if_db_exists: _, config_file_path: _, - #[cfg(feature = "analytics")] no_analytics, experimental_contains_filter, experimental_enable_metrics, experimental_search_queue_size, + experimental_drop_search_after, + experimental_nb_searches_per_core, experimental_logs_mode, experimental_enable_logs_route, experimental_replication_parameters, @@ -513,10 +529,7 @@ impl Opt { ); } - #[cfg(feature = "analytics")] - { - export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string()); - } + export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string()); export_to_env_if_not_present( MEILI_HTTP_PAYLOAD_SIZE_LIMIT, http_payload_size_limit.to_string(), @@ -559,6 +572,14 @@ impl Opt { MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE, experimental_search_queue_size.to_string(), ); + export_to_env_if_not_present( + MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER, + experimental_drop_search_after.to_string(), + ); + export_to_env_if_not_present( + MEILI_EXPERIMENTAL_NB_SEARCHES_PER_CORE, + experimental_nb_searches_per_core.to_string(), + ); export_to_env_if_not_present( MEILI_EXPERIMENTAL_LOGS_MODE, experimental_logs_mode.to_string(), @@ -890,6 +911,18 @@ fn default_dump_dir() -> PathBuf { PathBuf::from(DEFAULT_DUMP_DIR) } +fn default_experimental_search_queue_size() -> usize { + 1000 +} + +fn default_drop_search_after() -> NonZeroUsize { + NonZeroUsize::new(60).unwrap() +} + +fn default_nb_searches_per_core() -> NonZeroUsize { + NonZeroUsize::new(4).unwrap() +} + /// Indicates if a snapshot was scheduled, and if yes with which interval. #[derive(Debug, Default, Copy, Clone, Deserialize, Serialize)] pub enum ScheduleSnapshot { diff --git a/meilisearch/src/routes/api_key.rs b/crates/meilisearch/src/routes/api_key.rs similarity index 100% rename from meilisearch/src/routes/api_key.rs rename to crates/meilisearch/src/routes/api_key.rs diff --git a/meilisearch/src/routes/dump.rs b/crates/meilisearch/src/routes/dump.rs similarity index 90% rename from meilisearch/src/routes/dump.rs rename to crates/meilisearch/src/routes/dump.rs index 7f3cd06a5..c78dc4dad 100644 --- a/meilisearch/src/routes/dump.rs +++ b/crates/meilisearch/src/routes/dump.rs @@ -4,7 +4,6 @@ use index_scheduler::IndexScheduler; use meilisearch_auth::AuthController; use meilisearch_types::error::ResponseError; use meilisearch_types::tasks::KindWithContent; -use serde_json::json; use tracing::debug; use crate::analytics::Analytics; @@ -18,14 +17,16 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump)))); } +crate::empty_analytics!(DumpAnalytics, "Dump Created"); + pub async fn create_dump( index_scheduler: GuardedData, Data>, auth_controller: GuardedData, Data>, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { - analytics.publish("Dump Created".to_string(), json!({}), Some(&req)); + analytics.publish(DumpAnalytics::default(), &req); let task = KindWithContent::DumpCreation { keys: auth_controller.list_keys()?, diff --git a/meilisearch/src/routes/features.rs b/crates/meilisearch/src/routes/features.rs similarity index 73% rename from meilisearch/src/routes/features.rs rename to crates/meilisearch/src/routes/features.rs index bc656bdbb..5d93adc02 100644 --- a/meilisearch/src/routes/features.rs +++ b/crates/meilisearch/src/routes/features.rs @@ -6,10 +6,10 @@ use index_scheduler::IndexScheduler; use meilisearch_types::deserr::DeserrJsonError; use meilisearch_types::error::ResponseError; use meilisearch_types::keys::actions; -use serde_json::json; +use serde::Serialize; use tracing::debug; -use crate::analytics::Analytics; +use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; @@ -17,7 +17,7 @@ use crate::extractors::sequential_extractor::SeqHandler; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") - .route(web::get().to(SeqHandler(get_features))) + .route(web::get().to(get_features)) .route(web::patch().to(SeqHandler(patch_features))), ); } @@ -27,12 +27,9 @@ async fn get_features( ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_GET }>, Data, >, - req: HttpRequest, - analytics: Data, ) -> HttpResponse { let features = index_scheduler.features(); - analytics.publish("Experimental features Seen".to_string(), json!(null), Some(&req)); let features = features.runtime_features(); debug!(returns = ?features, "Get features"); HttpResponse::Ok().json(features) @@ -53,6 +50,35 @@ pub struct RuntimeTogglableFeatures { pub contains_filter: Option, } +#[derive(Serialize)] +pub struct PatchExperimentalFeatureAnalytics { + vector_store: bool, + metrics: bool, + logs_route: bool, + edit_documents_by_function: bool, + contains_filter: bool, +} + +impl Aggregate for PatchExperimentalFeatureAnalytics { + fn event_name(&self) -> &'static str { + "Experimental features Updated" + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { + vector_store: new.vector_store, + metrics: new.metrics, + logs_route: new.logs_route, + edit_documents_by_function: new.edit_documents_by_function, + contains_filter: new.contains_filter, + }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + async fn patch_features( index_scheduler: GuardedData< ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_UPDATE }>, @@ -60,7 +86,7 @@ async fn patch_features( >, new_features: AwebJson, req: HttpRequest, - analytics: Data, + analytics: Data, ) -> Result { let features = index_scheduler.features(); debug!(parameters = ?new_features, "Patch features"); @@ -89,15 +115,14 @@ async fn patch_features( } = new_features; analytics.publish( - "Experimental features Updated".to_string(), - json!({ - "vector_store": vector_store, - "metrics": metrics, - "logs_route": logs_route, - "edit_documents_by_function": edit_documents_by_function, - "contains_filter": contains_filter, - }), - Some(&req), + PatchExperimentalFeatureAnalytics { + vector_store, + metrics, + logs_route, + edit_documents_by_function, + contains_filter, + }, + &req, ); index_scheduler.put_runtime_features(new_features)?; debug!(returns = ?new_features, "Patch features"); diff --git a/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs similarity index 76% rename from meilisearch/src/routes/indexes/documents.rs rename to crates/meilisearch/src/routes/indexes/documents.rs index 85cf33c54..47f73ef42 100644 --- a/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1,4 +1,6 @@ +use std::collections::HashSet; use std::io::ErrorKind; +use std::marker::PhantomData; use actix_web::http::header::CONTENT_TYPE; use actix_web::web::Data; @@ -23,14 +25,14 @@ use meilisearch_types::tasks::KindWithContent; use meilisearch_types::{milli, Document, Index}; use mime::Mime; use once_cell::sync::Lazy; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; use tempfile::tempfile; use tokio::fs::File; use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufWriter}; use tracing::debug; -use crate::analytics::{Analytics, DocumentDeletionKind, DocumentFetchKind}; +use crate::analytics::{Aggregate, AggregateMethod, Analytics}; use crate::error::MeilisearchHttpError; use crate::error::PayloadError::ReceivePayload; use crate::extractors::authentication::policies::*; @@ -41,7 +43,7 @@ use crate::routes::{ get_task_id, is_dry_run, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT, }; use crate::search::{parse_filter, RetrieveVectors}; -use crate::Opt; +use crate::{aggregate_methods, Opt}; static ACCEPTED_CONTENT_TYPE: Lazy> = Lazy::new(|| { vec!["application/json".to_string(), "application/x-ndjson".to_string(), "text/csv".to_string()] @@ -100,12 +102,84 @@ pub struct GetDocument { retrieve_vectors: Param, } +aggregate_methods!( + DocumentsGET => "Documents Fetched GET", + DocumentsPOST => "Documents Fetched POST", +); + +#[derive(Serialize)] +pub struct DocumentsFetchAggregator { + // a call on ../documents/:doc_id + per_document_id: bool, + // if a filter was used + per_filter: bool, + + #[serde(rename = "vector.retrieve_vectors")] + retrieve_vectors: bool, + + // pagination + #[serde(rename = "pagination.max_limit")] + max_limit: usize, + #[serde(rename = "pagination.max_offset")] + max_offset: usize, + + marker: std::marker::PhantomData, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum DocumentFetchKind { + PerDocumentId { retrieve_vectors: bool }, + Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool }, +} + +impl DocumentsFetchAggregator { + pub fn from_query(query: &DocumentFetchKind) -> Self { + let (limit, offset, retrieve_vectors) = match query { + DocumentFetchKind::PerDocumentId { retrieve_vectors } => (1, 0, *retrieve_vectors), + DocumentFetchKind::Normal { limit, offset, retrieve_vectors, .. } => { + (*limit, *offset, *retrieve_vectors) + } + }; + + Self { + per_document_id: matches!(query, DocumentFetchKind::PerDocumentId { .. }), + per_filter: matches!(query, DocumentFetchKind::Normal { with_filter, .. } if *with_filter), + max_limit: limit, + max_offset: offset, + retrieve_vectors, + + marker: PhantomData, + } + } +} + +impl Aggregate for DocumentsFetchAggregator { + fn event_name(&self) -> &'static str { + Method::event_name() + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { + per_document_id: self.per_document_id | new.per_document_id, + per_filter: self.per_filter | new.per_filter, + retrieve_vectors: self.retrieve_vectors | new.retrieve_vectors, + max_limit: self.max_limit.max(new.max_limit), + max_offset: self.max_offset.max(new.max_offset), + marker: PhantomData, + }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + pub async fn get_document( index_scheduler: GuardedData, Data>, document_param: web::Path, params: AwebQueryParameter, req: HttpRequest, - analytics: web::Data, + analytics: web::Data, ) -> Result { let DocumentParam { index_uid, document_id } = document_param.into_inner(); debug!(parameters = ?params, "Get document"); @@ -117,8 +191,15 @@ pub async fn get_document( let features = index_scheduler.features(); let retrieve_vectors = RetrieveVectors::new(param_retrieve_vectors.0, features)?; - analytics.get_fetch_documents( - &DocumentFetchKind::PerDocumentId { retrieve_vectors: param_retrieve_vectors.0 }, + analytics.publish( + DocumentsFetchAggregator:: { + retrieve_vectors: param_retrieve_vectors.0, + per_document_id: true, + per_filter: false, + max_limit: 0, + max_offset: 0, + marker: PhantomData, + }, &req, ); @@ -129,17 +210,52 @@ pub async fn get_document( Ok(HttpResponse::Ok().json(document)) } +#[derive(Serialize)] +pub struct DocumentsDeletionAggregator { + per_document_id: bool, + clear_all: bool, + per_batch: bool, + per_filter: bool, +} + +impl Aggregate for DocumentsDeletionAggregator { + fn event_name(&self) -> &'static str { + "Documents Deleted" + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { + per_document_id: self.per_document_id | new.per_document_id, + clear_all: self.clear_all | new.clear_all, + per_batch: self.per_batch | new.per_batch, + per_filter: self.per_filter | new.per_filter, + }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + pub async fn delete_document( index_scheduler: GuardedData, Data>, path: web::Path, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { let DocumentParam { index_uid, document_id } = path.into_inner(); let index_uid = IndexUid::try_from(index_uid)?; - analytics.delete_documents(DocumentDeletionKind::PerDocumentId, &req); + analytics.publish( + DocumentsDeletionAggregator { + per_document_id: true, + clear_all: false, + per_batch: false, + per_filter: false, + }, + &req, + ); let task = KindWithContent::DocumentDeletion { index_uid: index_uid.to_string(), @@ -190,17 +306,19 @@ pub async fn documents_by_query_post( index_uid: web::Path, body: AwebJson, req: HttpRequest, - analytics: web::Data, + analytics: web::Data, ) -> Result { let body = body.into_inner(); debug!(parameters = ?body, "Get documents POST"); - analytics.post_fetch_documents( - &DocumentFetchKind::Normal { - with_filter: body.filter.is_some(), - limit: body.limit, - offset: body.offset, + analytics.publish( + DocumentsFetchAggregator:: { + per_filter: body.filter.is_some(), retrieve_vectors: body.retrieve_vectors, + max_limit: body.limit, + max_offset: body.offset, + per_document_id: false, + marker: PhantomData, }, &req, ); @@ -213,7 +331,7 @@ pub async fn get_documents( index_uid: web::Path, params: AwebQueryParameter, req: HttpRequest, - analytics: web::Data, + analytics: web::Data, ) -> Result { debug!(parameters = ?params, "Get documents GET"); @@ -235,12 +353,14 @@ pub async fn get_documents( filter, }; - analytics.get_fetch_documents( - &DocumentFetchKind::Normal { - with_filter: query.filter.is_some(), - limit: query.limit, - offset: query.offset, + analytics.publish( + DocumentsFetchAggregator:: { + per_filter: query.filter.is_some(), retrieve_vectors: query.retrieve_vectors, + max_limit: query.limit, + max_offset: query.offset, + per_document_id: false, + marker: PhantomData, }, &req, ); @@ -298,6 +418,39 @@ fn from_char_csv_delimiter( } } +aggregate_methods!( + Replaced => "Documents Added", + Updated => "Documents Updated", +); + +#[derive(Serialize)] +pub struct DocumentsAggregator { + payload_types: HashSet, + primary_key: HashSet, + index_creation: bool, + #[serde(skip)] + method: PhantomData, +} + +impl Aggregate for DocumentsAggregator { + fn event_name(&self) -> &'static str { + Method::event_name() + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { + payload_types: self.payload_types.union(&new.payload_types).cloned().collect(), + primary_key: self.primary_key.union(&new.primary_key).cloned().collect(), + index_creation: self.index_creation | new.index_creation, + method: PhantomData, + }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(self).unwrap_or_default() + } +} + pub async fn replace_documents( index_scheduler: GuardedData, Data>, index_uid: web::Path, @@ -305,16 +458,32 @@ pub async fn replace_documents( body: Payload, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; debug!(parameters = ?params, "Replace documents"); let params = params.into_inner(); - analytics.add_documents( - ¶ms, - index_scheduler.index_exists(&index_uid).map_or(true, |x| !x), + let mut content_types = HashSet::new(); + let content_type = req + .headers() + .get(CONTENT_TYPE) + .and_then(|s| s.to_str().ok()) + .unwrap_or("unknown") + .to_string(); + content_types.insert(content_type); + let mut primary_keys = HashSet::new(); + if let Some(primary_key) = params.primary_key.clone() { + primary_keys.insert(primary_key); + } + analytics.publish( + DocumentsAggregator:: { + payload_types: content_types, + primary_key: primary_keys, + index_creation: index_scheduler.index_exists(&index_uid).map_or(true, |x| !x), + method: PhantomData, + }, &req, ); @@ -346,16 +515,32 @@ pub async fn update_documents( body: Payload, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; let params = params.into_inner(); debug!(parameters = ?params, "Update documents"); - analytics.add_documents( - ¶ms, - index_scheduler.index_exists(&index_uid).map_or(true, |x| !x), + let mut content_types = HashSet::new(); + let content_type = req + .headers() + .get(CONTENT_TYPE) + .and_then(|s| s.to_str().ok()) + .unwrap_or("unknown") + .to_string(); + content_types.insert(content_type); + let mut primary_keys = HashSet::new(); + if let Some(primary_key) = params.primary_key.clone() { + primary_keys.insert(primary_key); + } + analytics.publish( + DocumentsAggregator:: { + payload_types: content_types, + primary_key: primary_keys, + index_creation: index_scheduler.index_exists(&index_uid).map_or(true, |x| !x), + method: PhantomData, + }, &req, ); @@ -524,12 +709,20 @@ pub async fn delete_documents_batch( body: web::Json>, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { debug!(parameters = ?body, "Delete documents by batch"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; - analytics.delete_documents(DocumentDeletionKind::PerBatch, &req); + analytics.publish( + DocumentsDeletionAggregator { + per_batch: true, + per_document_id: false, + clear_all: false, + per_filter: false, + }, + &req, + ); let ids = body .iter() @@ -562,14 +755,22 @@ pub async fn delete_documents_by_filter( body: AwebJson, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { debug!(parameters = ?body, "Delete documents by filter"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; let index_uid = index_uid.into_inner(); let filter = body.into_inner().filter; - analytics.delete_documents(DocumentDeletionKind::PerFilter, &req); + analytics.publish( + DocumentsDeletionAggregator { + per_filter: true, + per_document_id: false, + clear_all: false, + per_batch: false, + }, + &req, + ); // we ensure the filter is well formed before enqueuing it crate::search::parse_filter(&filter, Code::InvalidDocumentFilter, index_scheduler.features())? @@ -599,13 +800,41 @@ pub struct DocumentEditionByFunction { pub function: String, } +#[derive(Serialize)] +struct EditDocumentsByFunctionAggregator { + // Set to true if at least one request was filtered + filtered: bool, + // Set to true if at least one request contained a context + with_context: bool, + + index_creation: bool, +} + +impl Aggregate for EditDocumentsByFunctionAggregator { + fn event_name(&self) -> &'static str { + "Documents Edited By Function" + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { + filtered: self.filtered | new.filtered, + with_context: self.with_context | new.with_context, + index_creation: self.index_creation | new.index_creation, + }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + pub async fn edit_documents_by_function( index_scheduler: GuardedData, Data>, index_uid: web::Path, params: AwebJson, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { debug!(parameters = ?params, "Edit documents by function"); @@ -617,9 +846,12 @@ pub async fn edit_documents_by_function( let index_uid = index_uid.into_inner(); let params = params.into_inner(); - analytics.update_documents_by_function( - ¶ms, - index_scheduler.index(&index_uid).is_err(), + analytics.publish( + EditDocumentsByFunctionAggregator { + filtered: params.filter.is_some(), + with_context: params.context.is_some(), + index_creation: index_scheduler.index(&index_uid).is_err(), + }, &req, ); @@ -670,10 +902,18 @@ pub async fn clear_all_documents( index_uid: web::Path, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; - analytics.delete_documents(DocumentDeletionKind::ClearAll, &req); + analytics.publish( + DocumentsDeletionAggregator { + clear_all: true, + per_document_id: false, + per_batch: false, + per_filter: false, + }, + &req, + ); let task = KindWithContent::DocumentClear { index_uid: index_uid.to_string() }; let uid = get_task_id(&req, &opt)?; diff --git a/meilisearch/src/routes/indexes/facet_search.rs b/crates/meilisearch/src/routes/indexes/facet_search.rs similarity index 57% rename from meilisearch/src/routes/indexes/facet_search.rs rename to crates/meilisearch/src/routes/indexes/facet_search.rs index 1df80711d..99a4a4f28 100644 --- a/meilisearch/src/routes/indexes/facet_search.rs +++ b/crates/meilisearch/src/routes/indexes/facet_search.rs @@ -1,3 +1,5 @@ +use std::collections::{BinaryHeap, HashSet}; + use actix_web::web::Data; use actix_web::{web, HttpRequest, HttpResponse}; use deserr::actix_web::AwebJson; @@ -10,14 +12,15 @@ use meilisearch_types::locales::Locale; use serde_json::Value; use tracing::debug; -use crate::analytics::{Analytics, FacetSearchAggregator}; +use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::routes::indexes::search::search_kind; use crate::search::{ - add_search_rules, perform_facet_search, HybridQuery, MatchingStrategy, RankingScoreThreshold, - SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, - DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, + add_search_rules, perform_facet_search, FacetSearchResult, HybridQuery, MatchingStrategy, + RankingScoreThreshold, SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, + DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, + DEFAULT_SEARCH_OFFSET, }; use crate::search_queue::SearchQueue; @@ -53,20 +56,122 @@ pub struct FacetSearchQuery { pub locales: Option>, } +#[derive(Default)] +pub struct FacetSearchAggregator { + // requests + total_received: usize, + total_succeeded: usize, + time_spent: BinaryHeap, + + // The set of all facetNames that were used + facet_names: HashSet, + + // As there been any other parameter than the facetName or facetQuery ones? + additional_search_parameters_provided: bool, +} + +impl FacetSearchAggregator { + #[allow(clippy::field_reassign_with_default)] + pub fn from_query(query: &FacetSearchQuery) -> Self { + let FacetSearchQuery { + facet_query: _, + facet_name, + vector, + q, + filter, + matching_strategy, + attributes_to_search_on, + hybrid, + ranking_score_threshold, + locales, + } = query; + + Self { + total_received: 1, + facet_names: Some(facet_name.clone()).into_iter().collect(), + additional_search_parameters_provided: q.is_some() + || vector.is_some() + || filter.is_some() + || *matching_strategy != MatchingStrategy::default() + || attributes_to_search_on.is_some() + || hybrid.is_some() + || ranking_score_threshold.is_some() + || locales.is_some(), + ..Default::default() + } + } + + pub fn succeed(&mut self, result: &FacetSearchResult) { + let FacetSearchResult { facet_hits: _, facet_query: _, processing_time_ms } = result; + self.total_succeeded = 1; + self.time_spent.push(*processing_time_ms as usize); + } +} + +impl Aggregate for FacetSearchAggregator { + fn event_name(&self) -> &'static str { + "Facet Searched POST" + } + + fn aggregate(mut self: Box, new: Box) -> Box { + for time in new.time_spent { + self.time_spent.push(time); + } + + Box::new(Self { + total_received: self.total_received.saturating_add(new.total_received), + total_succeeded: self.total_succeeded.saturating_add(new.total_succeeded), + time_spent: self.time_spent, + facet_names: self.facet_names.union(&new.facet_names).cloned().collect(), + additional_search_parameters_provided: self.additional_search_parameters_provided + | new.additional_search_parameters_provided, + }) + } + + fn into_event(self: Box) -> serde_json::Value { + let Self { + total_received, + total_succeeded, + time_spent, + facet_names, + additional_search_parameters_provided, + } = *self; + // the index of the 99th percentage of value + let percentile_99th = 0.99 * (total_succeeded as f64 - 1.) + 1.; + // we get all the values in a sorted manner + let time_spent = time_spent.into_sorted_vec(); + // We are only interested by the slowest value of the 99th fastest results + let time_spent = time_spent.get(percentile_99th as usize); + + serde_json::json!({ + "requests": { + "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), + "total_succeeded": total_succeeded, + "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics + "total_received": total_received, + }, + "facets": { + "total_distinct_facet_count": facet_names.len(), + "additional_search_parameters_provided": additional_search_parameters_provided, + }, + }) + } +} + pub async fn search( index_scheduler: GuardedData, Data>, search_queue: Data, index_uid: web::Path, params: AwebJson, req: HttpRequest, - analytics: web::Data, + analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; let query = params.into_inner(); debug!(parameters = ?query, "Facet search"); - let mut aggregate = FacetSearchAggregator::from_query(&query, &req); + let mut aggregate = FacetSearchAggregator::from_query(&query); let facet_query = query.facet_query.clone(); let facet_name = query.facet_name.clone(); @@ -100,7 +205,7 @@ pub async fn search( if let Ok(ref search_result) = search_result { aggregate.succeed(search_result); } - analytics.post_facet_search(aggregate); + analytics.publish(aggregate, &req); let search_result = search_result?; diff --git a/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs similarity index 87% rename from meilisearch/src/routes/indexes/mod.rs rename to crates/meilisearch/src/routes/indexes/mod.rs index 35b747ccf..7d073ec5f 100644 --- a/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::convert::Infallible; use actix_web::web::Data; @@ -13,12 +14,11 @@ use meilisearch_types::index_uid::IndexUid; use meilisearch_types::milli::{self, FieldDistribution, Index}; use meilisearch_types::tasks::KindWithContent; use serde::Serialize; -use serde_json::json; use time::OffsetDateTime; use tracing::debug; use super::{get_task_id, Pagination, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT}; -use crate::analytics::Analytics; +use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::{AuthenticationError, GuardedData}; use crate::extractors::sequential_extractor::SeqHandler; @@ -28,8 +28,11 @@ use crate::Opt; pub mod documents; pub mod facet_search; pub mod search; +mod search_analytics; pub mod settings; +mod settings_analytics; pub mod similar; +mod similar_analytics; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( @@ -123,12 +126,31 @@ pub struct IndexCreateRequest { primary_key: Option, } +#[derive(Serialize)] +struct IndexCreatedAggregate { + primary_key: BTreeSet, +} + +impl Aggregate for IndexCreatedAggregate { + fn event_name(&self) -> &'static str { + "Index Created" + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { primary_key: self.primary_key.union(&new.primary_key).cloned().collect() }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + pub async fn create_index( index_scheduler: GuardedData, Data>, body: AwebJson, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { debug!(parameters = ?body, "Create index"); let IndexCreateRequest { primary_key, uid } = body.into_inner(); @@ -136,9 +158,8 @@ pub async fn create_index( let allow_index_creation = index_scheduler.filters().allow_index_creation(&uid); if allow_index_creation { analytics.publish( - "Index Created".to_string(), - json!({ "primary_key": primary_key }), - Some(&req), + IndexCreatedAggregate { primary_key: primary_key.iter().cloned().collect() }, + &req, ); let task = KindWithContent::IndexCreation { index_uid: uid.to_string(), primary_key }; @@ -194,21 +215,38 @@ pub async fn get_index( Ok(HttpResponse::Ok().json(index_view)) } +#[derive(Serialize)] +struct IndexUpdatedAggregate { + primary_key: BTreeSet, +} + +impl Aggregate for IndexUpdatedAggregate { + fn event_name(&self) -> &'static str { + "Index Updated" + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { primary_key: self.primary_key.union(&new.primary_key).cloned().collect() }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} pub async fn update_index( index_scheduler: GuardedData, Data>, index_uid: web::Path, body: AwebJson, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { debug!(parameters = ?body, "Update index"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; let body = body.into_inner(); analytics.publish( - "Index Updated".to_string(), - json!({ "primary_key": body.primary_key }), - Some(&req), + IndexUpdatedAggregate { primary_key: body.primary_key.iter().cloned().collect() }, + &req, ); let task = KindWithContent::IndexUpdate { diff --git a/meilisearch/src/routes/indexes/search.rs b/crates/meilisearch/src/routes/indexes/search.rs similarity index 97% rename from meilisearch/src/routes/indexes/search.rs rename to crates/meilisearch/src/routes/indexes/search.rs index 6a8eee521..2f5cb4a36 100644 --- a/meilisearch/src/routes/indexes/search.rs +++ b/crates/meilisearch/src/routes/indexes/search.rs @@ -13,12 +13,13 @@ use meilisearch_types::serde_cs::vec::CS; use serde_json::Value; use tracing::debug; -use crate::analytics::{Analytics, SearchAggregator}; +use crate::analytics::Analytics; use crate::error::MeilisearchHttpError; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS; +use crate::routes::indexes::search_analytics::{SearchAggregator, SearchGET, SearchPOST}; use crate::search::{ add_search_rules, perform_search, HybridQuery, MatchingStrategy, RankingScoreThreshold, RetrieveVectors, SearchKind, SearchQuery, SemanticRatio, DEFAULT_CROP_LENGTH, @@ -225,7 +226,7 @@ pub async fn search_with_url_query( index_uid: web::Path, params: AwebQueryParameter, req: HttpRequest, - analytics: web::Data, + analytics: web::Data, ) -> Result { debug!(parameters = ?params, "Search get"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; @@ -237,7 +238,7 @@ pub async fn search_with_url_query( add_search_rules(&mut query.filter, search_rules); } - let mut aggregate = SearchAggregator::from_query(&query, &req); + let mut aggregate = SearchAggregator::::from_query(&query); let index = index_scheduler.index(&index_uid)?; let features = index_scheduler.features(); @@ -254,7 +255,7 @@ pub async fn search_with_url_query( if let Ok(ref search_result) = search_result { aggregate.succeed(search_result); } - analytics.get_search(aggregate); + analytics.publish(aggregate, &req); let search_result = search_result?; @@ -268,7 +269,7 @@ pub async fn search_with_post( index_uid: web::Path, params: AwebJson, req: HttpRequest, - analytics: web::Data, + analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; @@ -280,7 +281,7 @@ pub async fn search_with_post( add_search_rules(&mut query.filter, search_rules); } - let mut aggregate = SearchAggregator::from_query(&query, &req); + let mut aggregate = SearchAggregator::::from_query(&query); let index = index_scheduler.index(&index_uid)?; @@ -302,7 +303,7 @@ pub async fn search_with_post( MEILISEARCH_DEGRADED_SEARCH_REQUESTS.inc(); } } - analytics.post_search(aggregate); + analytics.publish(aggregate, &req); let search_result = search_result?; diff --git a/crates/meilisearch/src/routes/indexes/search_analytics.rs b/crates/meilisearch/src/routes/indexes/search_analytics.rs new file mode 100644 index 000000000..b16e2636e --- /dev/null +++ b/crates/meilisearch/src/routes/indexes/search_analytics.rs @@ -0,0 +1,483 @@ +use std::collections::{BTreeSet, BinaryHeap, HashMap}; + +use meilisearch_types::locales::Locale; +use once_cell::sync::Lazy; +use regex::Regex; +use serde_json::{json, Value}; + +use crate::aggregate_methods; +use crate::analytics::{Aggregate, AggregateMethod}; +use crate::search::{ + SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, + DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, + DEFAULT_SEMANTIC_RATIO, +}; + +aggregate_methods!( + SearchGET => "Documents Searched GET", + SearchPOST => "Documents Searched POST", +); + +#[derive(Default)] +pub struct SearchAggregator { + // requests + total_received: usize, + total_succeeded: usize, + total_degraded: usize, + total_used_negative_operator: usize, + time_spent: BinaryHeap, + + // sort + sort_with_geo_point: bool, + // every time a request has a filter, this field must be incremented by the number of terms it contains + sort_sum_of_criteria_terms: usize, + // every time a request has a filter, this field must be incremented by one + sort_total_number_of_criteria: usize, + + // distinct + distinct: bool, + + // filter + filter_with_geo_radius: bool, + filter_with_geo_bounding_box: bool, + // every time a request has a filter, this field must be incremented by the number of terms it contains + filter_sum_of_criteria_terms: usize, + // every time a request has a filter, this field must be incremented by one + filter_total_number_of_criteria: usize, + used_syntax: HashMap, + + // attributes_to_search_on + // every time a search is done using attributes_to_search_on + attributes_to_search_on_total_number_of_uses: usize, + + // q + // The maximum number of terms in a q request + max_terms_number: usize, + + // vector + // The maximum number of floats in a vector request + max_vector_size: usize, + // Whether the semantic ratio passed to a hybrid search equals the default ratio. + semantic_ratio: bool, + hybrid: bool, + retrieve_vectors: bool, + + // every time a search is done, we increment the counter linked to the used settings + matching_strategy: HashMap, + + // List of the unique Locales passed as parameter + locales: BTreeSet, + + // pagination + max_limit: usize, + max_offset: usize, + finite_pagination: usize, + + // formatting + max_attributes_to_retrieve: usize, + max_attributes_to_highlight: usize, + highlight_pre_tag: bool, + highlight_post_tag: bool, + max_attributes_to_crop: usize, + crop_marker: bool, + show_matches_position: bool, + crop_length: bool, + + // facets + facets_sum_of_terms: usize, + facets_total_number_of_facets: usize, + + // scoring + show_ranking_score: bool, + show_ranking_score_details: bool, + ranking_score_threshold: bool, + + marker: std::marker::PhantomData, +} + +impl SearchAggregator { + #[allow(clippy::field_reassign_with_default)] + pub fn from_query(query: &SearchQuery) -> Self { + let SearchQuery { + q, + vector, + offset, + limit, + page, + hits_per_page, + attributes_to_retrieve: _, + retrieve_vectors, + attributes_to_crop: _, + crop_length, + attributes_to_highlight: _, + show_matches_position, + show_ranking_score, + show_ranking_score_details, + filter, + sort, + distinct, + facets: _, + highlight_pre_tag, + highlight_post_tag, + crop_marker, + matching_strategy, + attributes_to_search_on, + hybrid, + ranking_score_threshold, + locales, + } = query; + + let mut ret = Self::default(); + + ret.total_received = 1; + + if let Some(ref sort) = sort { + ret.sort_total_number_of_criteria = 1; + ret.sort_with_geo_point = sort.iter().any(|s| s.contains("_geoPoint(")); + ret.sort_sum_of_criteria_terms = sort.len(); + } + + ret.distinct = distinct.is_some(); + + if let Some(ref filter) = filter { + static RE: Lazy = Lazy::new(|| Regex::new("AND | OR").unwrap()); + ret.filter_total_number_of_criteria = 1; + + let syntax = match filter { + Value::String(_) => "string".to_string(), + Value::Array(values) => { + if values.iter().map(|v| v.to_string()).any(|s| RE.is_match(&s)) { + "mixed".to_string() + } else { + "array".to_string() + } + } + _ => "none".to_string(), + }; + // convert the string to a HashMap + ret.used_syntax.insert(syntax, 1); + + let stringified_filters = filter.to_string(); + ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius("); + ret.filter_with_geo_bounding_box = stringified_filters.contains("_geoBoundingBox("); + ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count(); + } + + // attributes_to_search_on + if attributes_to_search_on.is_some() { + ret.attributes_to_search_on_total_number_of_uses = 1; + } + + if let Some(ref q) = q { + ret.max_terms_number = q.split_whitespace().count(); + } + + if let Some(ref vector) = vector { + ret.max_vector_size = vector.len(); + } + ret.retrieve_vectors |= retrieve_vectors; + + if query.is_finite_pagination() { + let limit = hits_per_page.unwrap_or_else(DEFAULT_SEARCH_LIMIT); + ret.max_limit = limit; + ret.max_offset = page.unwrap_or(1).saturating_sub(1) * limit; + ret.finite_pagination = 1; + } else { + ret.max_limit = *limit; + ret.max_offset = *offset; + ret.finite_pagination = 0; + } + + ret.matching_strategy.insert(format!("{:?}", matching_strategy), 1); + + if let Some(locales) = locales { + ret.locales = locales.iter().copied().collect(); + } + + ret.highlight_pre_tag = *highlight_pre_tag != DEFAULT_HIGHLIGHT_PRE_TAG(); + ret.highlight_post_tag = *highlight_post_tag != DEFAULT_HIGHLIGHT_POST_TAG(); + ret.crop_marker = *crop_marker != DEFAULT_CROP_MARKER(); + ret.crop_length = *crop_length != DEFAULT_CROP_LENGTH(); + ret.show_matches_position = *show_matches_position; + + ret.show_ranking_score = *show_ranking_score; + ret.show_ranking_score_details = *show_ranking_score_details; + ret.ranking_score_threshold = ranking_score_threshold.is_some(); + + if let Some(hybrid) = hybrid { + ret.semantic_ratio = hybrid.semantic_ratio != DEFAULT_SEMANTIC_RATIO(); + ret.hybrid = true; + } + + ret + } + + pub fn succeed(&mut self, result: &SearchResult) { + let SearchResult { + hits: _, + query: _, + processing_time_ms, + hits_info: _, + semantic_hit_count: _, + facet_distribution: _, + facet_stats: _, + degraded, + used_negative_operator, + } = result; + + self.total_succeeded = self.total_succeeded.saturating_add(1); + if *degraded { + self.total_degraded = self.total_degraded.saturating_add(1); + } + if *used_negative_operator { + self.total_used_negative_operator = self.total_used_negative_operator.saturating_add(1); + } + self.time_spent.push(*processing_time_ms as usize); + } +} + +impl Aggregate for SearchAggregator { + fn event_name(&self) -> &'static str { + Method::event_name() + } + + fn aggregate(mut self: Box, new: Box) -> Box { + let Self { + total_received, + total_succeeded, + mut time_spent, + sort_with_geo_point, + sort_sum_of_criteria_terms, + sort_total_number_of_criteria, + distinct, + filter_with_geo_radius, + filter_with_geo_bounding_box, + filter_sum_of_criteria_terms, + filter_total_number_of_criteria, + used_syntax, + attributes_to_search_on_total_number_of_uses, + max_terms_number, + max_vector_size, + retrieve_vectors, + matching_strategy, + max_limit, + max_offset, + finite_pagination, + max_attributes_to_retrieve, + max_attributes_to_highlight, + highlight_pre_tag, + highlight_post_tag, + max_attributes_to_crop, + crop_marker, + show_matches_position, + crop_length, + facets_sum_of_terms, + facets_total_number_of_facets, + show_ranking_score, + show_ranking_score_details, + semantic_ratio, + hybrid, + total_degraded, + total_used_negative_operator, + ranking_score_threshold, + mut locales, + marker: _, + } = *new; + + // request + self.total_received = self.total_received.saturating_add(total_received); + self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded); + self.total_degraded = self.total_degraded.saturating_add(total_degraded); + self.total_used_negative_operator = + self.total_used_negative_operator.saturating_add(total_used_negative_operator); + self.time_spent.append(&mut time_spent); + + // sort + self.sort_with_geo_point |= sort_with_geo_point; + self.sort_sum_of_criteria_terms = + self.sort_sum_of_criteria_terms.saturating_add(sort_sum_of_criteria_terms); + self.sort_total_number_of_criteria = + self.sort_total_number_of_criteria.saturating_add(sort_total_number_of_criteria); + + // distinct + self.distinct |= distinct; + + // filter + self.filter_with_geo_radius |= filter_with_geo_radius; + self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box; + self.filter_sum_of_criteria_terms = + self.filter_sum_of_criteria_terms.saturating_add(filter_sum_of_criteria_terms); + self.filter_total_number_of_criteria = + self.filter_total_number_of_criteria.saturating_add(filter_total_number_of_criteria); + for (key, value) in used_syntax.into_iter() { + let used_syntax = self.used_syntax.entry(key).or_insert(0); + *used_syntax = used_syntax.saturating_add(value); + } + + // attributes_to_search_on + self.attributes_to_search_on_total_number_of_uses = self + .attributes_to_search_on_total_number_of_uses + .saturating_add(attributes_to_search_on_total_number_of_uses); + + // q + self.max_terms_number = self.max_terms_number.max(max_terms_number); + + // vector + self.max_vector_size = self.max_vector_size.max(max_vector_size); + self.retrieve_vectors |= retrieve_vectors; + self.semantic_ratio |= semantic_ratio; + self.hybrid |= hybrid; + + // pagination + self.max_limit = self.max_limit.max(max_limit); + self.max_offset = self.max_offset.max(max_offset); + self.finite_pagination += finite_pagination; + + // formatting + self.max_attributes_to_retrieve = + self.max_attributes_to_retrieve.max(max_attributes_to_retrieve); + self.max_attributes_to_highlight = + self.max_attributes_to_highlight.max(max_attributes_to_highlight); + self.highlight_pre_tag |= highlight_pre_tag; + self.highlight_post_tag |= highlight_post_tag; + self.max_attributes_to_crop = self.max_attributes_to_crop.max(max_attributes_to_crop); + self.crop_marker |= crop_marker; + self.show_matches_position |= show_matches_position; + self.crop_length |= crop_length; + + // facets + self.facets_sum_of_terms = self.facets_sum_of_terms.saturating_add(facets_sum_of_terms); + self.facets_total_number_of_facets = + self.facets_total_number_of_facets.saturating_add(facets_total_number_of_facets); + + // matching strategy + for (key, value) in matching_strategy.into_iter() { + let matching_strategy = self.matching_strategy.entry(key).or_insert(0); + *matching_strategy = matching_strategy.saturating_add(value); + } + + // scoring + self.show_ranking_score |= show_ranking_score; + self.show_ranking_score_details |= show_ranking_score_details; + self.ranking_score_threshold |= ranking_score_threshold; + + // locales + self.locales.append(&mut locales); + + self + } + + fn into_event(self: Box) -> serde_json::Value { + let Self { + total_received, + total_succeeded, + time_spent, + sort_with_geo_point, + sort_sum_of_criteria_terms, + sort_total_number_of_criteria, + distinct, + filter_with_geo_radius, + filter_with_geo_bounding_box, + filter_sum_of_criteria_terms, + filter_total_number_of_criteria, + used_syntax, + attributes_to_search_on_total_number_of_uses, + max_terms_number, + max_vector_size, + retrieve_vectors, + matching_strategy, + max_limit, + max_offset, + finite_pagination, + max_attributes_to_retrieve, + max_attributes_to_highlight, + highlight_pre_tag, + highlight_post_tag, + max_attributes_to_crop, + crop_marker, + show_matches_position, + crop_length, + facets_sum_of_terms, + facets_total_number_of_facets, + show_ranking_score, + show_ranking_score_details, + semantic_ratio, + hybrid, + total_degraded, + total_used_negative_operator, + ranking_score_threshold, + locales, + marker: _, + } = *self; + + // we get all the values in a sorted manner + let time_spent = time_spent.into_sorted_vec(); + // the index of the 99th percentage of value + let percentile_99th = time_spent.len() * 99 / 100; + // We are only interested by the slowest value of the 99th fastest results + let time_spent = time_spent.get(percentile_99th); + + json!({ + "requests": { + "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), + "total_succeeded": total_succeeded, + "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics + "total_received": total_received, + "total_degraded": total_degraded, + "total_used_negative_operator": total_used_negative_operator, + }, + "sort": { + "with_geoPoint": sort_with_geo_point, + "avg_criteria_number": format!("{:.2}", sort_sum_of_criteria_terms as f64 / sort_total_number_of_criteria as f64), + }, + "distinct": distinct, + "filter": { + "with_geoRadius": filter_with_geo_radius, + "with_geoBoundingBox": filter_with_geo_bounding_box, + "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64), + "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), + }, + "attributes_to_search_on": { + "total_number_of_uses": attributes_to_search_on_total_number_of_uses, + }, + "q": { + "max_terms_number": max_terms_number, + }, + "vector": { + "max_vector_size": max_vector_size, + "retrieve_vectors": retrieve_vectors, + }, + "hybrid": { + "enabled": hybrid, + "semantic_ratio": semantic_ratio, + }, + "pagination": { + "max_limit": max_limit, + "max_offset": max_offset, + "most_used_navigation": if finite_pagination > (total_received / 2) { "exhaustive" } else { "estimated" }, + }, + "formatting": { + "max_attributes_to_retrieve": max_attributes_to_retrieve, + "max_attributes_to_highlight": max_attributes_to_highlight, + "highlight_pre_tag": highlight_pre_tag, + "highlight_post_tag": highlight_post_tag, + "max_attributes_to_crop": max_attributes_to_crop, + "crop_marker": crop_marker, + "show_matches_position": show_matches_position, + "crop_length": crop_length, + }, + "facets": { + "avg_facets_number": format!("{:.2}", facets_sum_of_terms as f64 / facets_total_number_of_facets as f64), + }, + "matching_strategy": { + "most_used_strategy": matching_strategy.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), + }, + "locales": locales, + "scoring": { + "show_ranking_score": show_ranking_score, + "show_ranking_score_details": show_ranking_score_details, + "ranking_score_threshold": ranking_score_threshold, + }, + }) + } +} diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs new file mode 100644 index 000000000..a9d8d3053 --- /dev/null +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -0,0 +1,532 @@ +use actix_web::web::Data; +use actix_web::{web, HttpRequest, HttpResponse}; +use deserr::actix_web::AwebJson; +use index_scheduler::IndexScheduler; +use meilisearch_types::deserr::DeserrJsonError; +use meilisearch_types::error::ResponseError; +use meilisearch_types::index_uid::IndexUid; +use meilisearch_types::milli::update::Setting; +use meilisearch_types::settings::{settings, SecretPolicy, Settings, Unchecked}; +use meilisearch_types::tasks::KindWithContent; +use tracing::debug; + +use super::settings_analytics::*; +use crate::analytics::Analytics; +use crate::extractors::authentication::policies::*; +use crate::extractors::authentication::GuardedData; +use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView}; +use crate::Opt; + +#[macro_export] +macro_rules! make_setting_route { + ($route:literal, $update_verb:ident, $type:ty, $err_ty:ty, $attr:ident, $camelcase_attr:literal, $analytics:ident) => { + pub mod $attr { + use actix_web::web::Data; + use actix_web::{web, HttpRequest, HttpResponse, Resource}; + use index_scheduler::IndexScheduler; + use meilisearch_types::error::ResponseError; + use meilisearch_types::index_uid::IndexUid; + use meilisearch_types::milli::update::Setting; + use meilisearch_types::settings::{settings, Settings}; + use meilisearch_types::tasks::KindWithContent; + use tracing::debug; + use $crate::analytics::Analytics; + use $crate::extractors::authentication::policies::*; + use $crate::extractors::authentication::GuardedData; + use $crate::extractors::sequential_extractor::SeqHandler; + use $crate::Opt; + use $crate::routes::{is_dry_run, get_task_id, SummarizedTaskView}; + + pub async fn delete( + index_scheduler: GuardedData< + ActionPolicy<{ actions::SETTINGS_UPDATE }>, + Data, + >, + index_uid: web::Path, + req: HttpRequest, + opt: web::Data, + ) -> Result { + let index_uid = IndexUid::try_from(index_uid.into_inner())?; + + let new_settings = Settings { $attr: Setting::Reset.into(), ..Default::default() }; + + let allow_index_creation = + index_scheduler.filters().allow_index_creation(&index_uid); + + let task = KindWithContent::SettingsUpdate { + index_uid: index_uid.to_string(), + new_settings: Box::new(new_settings), + is_deletion: true, + allow_index_creation, + }; + let uid = get_task_id(&req, &opt)?; + let dry_run = is_dry_run(&req, &opt)?; + let task: SummarizedTaskView = + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) + .await?? + .into(); + + debug!(returns = ?task, "Delete settings"); + Ok(HttpResponse::Accepted().json(task)) + } + + pub async fn update( + index_scheduler: GuardedData< + ActionPolicy<{ actions::SETTINGS_UPDATE }>, + Data, + >, + index_uid: actix_web::web::Path, + body: deserr::actix_web::AwebJson, $err_ty>, + req: HttpRequest, + opt: web::Data, + analytics: web::Data, + ) -> std::result::Result { + let index_uid = IndexUid::try_from(index_uid.into_inner())?; + + let body = body.into_inner(); + debug!(parameters = ?body, "Update settings"); + + #[allow(clippy::redundant_closure_call)] + analytics.publish( + $crate::routes::indexes::settings_analytics::$analytics::new(body.as_ref()).into_settings(), + &req, + ); + + let new_settings = Settings { + $attr: match body { + Some(inner_body) => Setting::Set(inner_body).into(), + None => Setting::Reset.into(), + }, + ..Default::default() + }; + + let new_settings = $crate::routes::indexes::settings::validate_settings( + new_settings, + &index_scheduler, + )?; + + let allow_index_creation = + index_scheduler.filters().allow_index_creation(&index_uid); + + let task = KindWithContent::SettingsUpdate { + index_uid: index_uid.to_string(), + new_settings: Box::new(new_settings), + is_deletion: false, + allow_index_creation, + }; + let uid = get_task_id(&req, &opt)?; + let dry_run = is_dry_run(&req, &opt)?; + let task: SummarizedTaskView = + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) + .await?? + .into(); + + debug!(returns = ?task, "Update settings"); + Ok(HttpResponse::Accepted().json(task)) + } + + pub async fn get( + index_scheduler: GuardedData< + ActionPolicy<{ actions::SETTINGS_GET }>, + Data, + >, + index_uid: actix_web::web::Path, + ) -> std::result::Result { + let index_uid = IndexUid::try_from(index_uid.into_inner())?; + + let index = index_scheduler.index(&index_uid)?; + let rtxn = index.read_txn()?; + let settings = settings(&index, &rtxn, meilisearch_types::settings::SecretPolicy::HideSecrets)?; + + debug!(returns = ?settings, "Update settings"); + + Ok(HttpResponse::Ok().json(settings.$attr)) + } + + pub fn resources() -> Resource { + Resource::new($route) + .route(web::get().to(SeqHandler(get))) + .route(web::$update_verb().to(SeqHandler(update))) + .route(web::delete().to(SeqHandler(delete))) + } + } + }; +} + +make_setting_route!( + "/filterable-attributes", + put, + std::collections::BTreeSet, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsFilterableAttributes, + >, + filterable_attributes, + "filterableAttributes", + FilterableAttributesAnalytics +); + +make_setting_route!( + "/sortable-attributes", + put, + std::collections::BTreeSet, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsSortableAttributes, + >, + sortable_attributes, + "sortableAttributes", + SortableAttributesAnalytics +); + +make_setting_route!( + "/displayed-attributes", + put, + Vec, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsDisplayedAttributes, + >, + displayed_attributes, + "displayedAttributes", + DisplayedAttributesAnalytics +); + +make_setting_route!( + "/typo-tolerance", + patch, + meilisearch_types::settings::TypoSettings, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsTypoTolerance, + >, + typo_tolerance, + "typoTolerance", + TypoToleranceAnalytics +); + +make_setting_route!( + "/searchable-attributes", + put, + Vec, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsSearchableAttributes, + >, + searchable_attributes, + "searchableAttributes", + SearchableAttributesAnalytics +); + +make_setting_route!( + "/stop-words", + put, + std::collections::BTreeSet, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsStopWords, + >, + stop_words, + "stopWords", + StopWordsAnalytics +); + +make_setting_route!( + "/non-separator-tokens", + put, + std::collections::BTreeSet, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsNonSeparatorTokens, + >, + non_separator_tokens, + "nonSeparatorTokens", + NonSeparatorTokensAnalytics +); + +make_setting_route!( + "/separator-tokens", + put, + std::collections::BTreeSet, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsSeparatorTokens, + >, + separator_tokens, + "separatorTokens", + SeparatorTokensAnalytics +); + +make_setting_route!( + "/dictionary", + put, + std::collections::BTreeSet, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsDictionary, + >, + dictionary, + "dictionary", + DictionaryAnalytics +); + +make_setting_route!( + "/synonyms", + put, + std::collections::BTreeMap>, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsSynonyms, + >, + synonyms, + "synonyms", + SynonymsAnalytics +); + +make_setting_route!( + "/distinct-attribute", + put, + String, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsDistinctAttribute, + >, + distinct_attribute, + "distinctAttribute", + DistinctAttributeAnalytics +); + +make_setting_route!( + "/proximity-precision", + put, + meilisearch_types::settings::ProximityPrecisionView, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsProximityPrecision, + >, + proximity_precision, + "proximityPrecision", + ProximityPrecisionAnalytics +); + +make_setting_route!( + "/localized-attributes", + put, + Vec, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsLocalizedAttributes, + >, + localized_attributes, + "localizedAttributes", + LocalesAnalytics +); + +make_setting_route!( + "/ranking-rules", + put, + Vec, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsRankingRules, + >, + ranking_rules, + "rankingRules", + RankingRulesAnalytics +); + +make_setting_route!( + "/faceting", + patch, + meilisearch_types::settings::FacetingSettings, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsFaceting, + >, + faceting, + "faceting", + FacetingAnalytics +); + +make_setting_route!( + "/pagination", + patch, + meilisearch_types::settings::PaginationSettings, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsPagination, + >, + pagination, + "pagination", + PaginationAnalytics +); + +make_setting_route!( + "/embedders", + patch, + std::collections::BTreeMap>, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsEmbedders, + >, + embedders, + "embedders", + EmbeddersAnalytics +); + +make_setting_route!( + "/search-cutoff-ms", + put, + u64, + meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsSearchCutoffMs, + >, + search_cutoff_ms, + "searchCutoffMs", + SearchCutoffMsAnalytics +); + +macro_rules! generate_configure { + ($($mod:ident),*) => { + pub fn configure(cfg: &mut web::ServiceConfig) { + use crate::extractors::sequential_extractor::SeqHandler; + cfg.service( + web::resource("") + .route(web::patch().to(SeqHandler(update_all))) + .route(web::get().to(SeqHandler(get_all))) + .route(web::delete().to(SeqHandler(delete_all)))) + $(.service($mod::resources()))*; + } + }; +} + +generate_configure!( + filterable_attributes, + sortable_attributes, + displayed_attributes, + localized_attributes, + searchable_attributes, + distinct_attribute, + proximity_precision, + stop_words, + separator_tokens, + non_separator_tokens, + dictionary, + synonyms, + ranking_rules, + typo_tolerance, + pagination, + faceting, + embedders, + search_cutoff_ms +); + +pub async fn update_all( + index_scheduler: GuardedData, Data>, + index_uid: web::Path, + body: AwebJson, DeserrJsonError>, + req: HttpRequest, + opt: web::Data, + analytics: web::Data, +) -> Result { + let index_uid = IndexUid::try_from(index_uid.into_inner())?; + + let new_settings = body.into_inner(); + debug!(parameters = ?new_settings, "Update all settings"); + let new_settings = validate_settings(new_settings, &index_scheduler)?; + + analytics.publish( + SettingsAnalytics { + ranking_rules: RankingRulesAnalytics::new(new_settings.ranking_rules.as_ref().set()), + searchable_attributes: SearchableAttributesAnalytics::new( + new_settings.searchable_attributes.as_ref().set(), + ), + displayed_attributes: DisplayedAttributesAnalytics::new( + new_settings.displayed_attributes.as_ref().set(), + ), + sortable_attributes: SortableAttributesAnalytics::new( + new_settings.sortable_attributes.as_ref().set(), + ), + filterable_attributes: FilterableAttributesAnalytics::new( + new_settings.filterable_attributes.as_ref().set(), + ), + distinct_attribute: DistinctAttributeAnalytics::new( + new_settings.distinct_attribute.as_ref().set(), + ), + proximity_precision: ProximityPrecisionAnalytics::new( + new_settings.proximity_precision.as_ref().set(), + ), + typo_tolerance: TypoToleranceAnalytics::new(new_settings.typo_tolerance.as_ref().set()), + faceting: FacetingAnalytics::new(new_settings.faceting.as_ref().set()), + pagination: PaginationAnalytics::new(new_settings.pagination.as_ref().set()), + stop_words: StopWordsAnalytics::new(new_settings.stop_words.as_ref().set()), + synonyms: SynonymsAnalytics::new(new_settings.synonyms.as_ref().set()), + embedders: EmbeddersAnalytics::new(new_settings.embedders.as_ref().set()), + search_cutoff_ms: SearchCutoffMsAnalytics::new( + new_settings.search_cutoff_ms.as_ref().set(), + ), + locales: LocalesAnalytics::new(new_settings.localized_attributes.as_ref().set()), + dictionary: DictionaryAnalytics::new(new_settings.dictionary.as_ref().set()), + separator_tokens: SeparatorTokensAnalytics::new( + new_settings.separator_tokens.as_ref().set(), + ), + non_separator_tokens: NonSeparatorTokensAnalytics::new( + new_settings.non_separator_tokens.as_ref().set(), + ), + }, + &req, + ); + + let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid); + let index_uid = IndexUid::try_from(index_uid.into_inner())?.into_inner(); + let task = KindWithContent::SettingsUpdate { + index_uid, + new_settings: Box::new(new_settings), + is_deletion: false, + allow_index_creation, + }; + let uid = get_task_id(&req, &opt)?; + let dry_run = is_dry_run(&req, &opt)?; + let task: SummarizedTaskView = + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) + .await?? + .into(); + + debug!(returns = ?task, "Update all settings"); + Ok(HttpResponse::Accepted().json(task)) +} + +pub async fn get_all( + index_scheduler: GuardedData, Data>, + index_uid: web::Path, +) -> Result { + let index_uid = IndexUid::try_from(index_uid.into_inner())?; + + let index = index_scheduler.index(&index_uid)?; + let rtxn = index.read_txn()?; + let new_settings = settings(&index, &rtxn, SecretPolicy::HideSecrets)?; + debug!(returns = ?new_settings, "Get all settings"); + Ok(HttpResponse::Ok().json(new_settings)) +} + +pub async fn delete_all( + index_scheduler: GuardedData, Data>, + index_uid: web::Path, + req: HttpRequest, + opt: web::Data, +) -> Result { + let index_uid = IndexUid::try_from(index_uid.into_inner())?; + + let new_settings = Settings::cleared().into_unchecked(); + + let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid); + let index_uid = IndexUid::try_from(index_uid.into_inner())?.into_inner(); + let task = KindWithContent::SettingsUpdate { + index_uid, + new_settings: Box::new(new_settings), + is_deletion: true, + allow_index_creation, + }; + let uid = get_task_id(&req, &opt)?; + let dry_run = is_dry_run(&req, &opt)?; + let task: SummarizedTaskView = + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) + .await?? + .into(); + + debug!(returns = ?task, "Delete all settings"); + Ok(HttpResponse::Accepted().json(task)) +} + +fn validate_settings( + settings: Settings, + index_scheduler: &IndexScheduler, +) -> Result, ResponseError> { + if matches!(settings.embedders, Setting::Set(_)) { + index_scheduler.features().check_vector("Passing `embedders` in settings")? + } + Ok(settings.validate()?) +} diff --git a/crates/meilisearch/src/routes/indexes/settings_analytics.rs b/crates/meilisearch/src/routes/indexes/settings_analytics.rs new file mode 100644 index 000000000..32bddcbdd --- /dev/null +++ b/crates/meilisearch/src/routes/indexes/settings_analytics.rs @@ -0,0 +1,622 @@ +//! All the structures used to make the analytics on the settings works. +//! The signatures of the `new` functions are not very rust idiomatic because they must match the types received +//! through the sub-settings route directly without any manipulation. +//! This is why we often use a `Option<&Vec<_>>` instead of a `Option<&[_]>`. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use meilisearch_types::facet_values_sort::FacetValuesSort; +use meilisearch_types::locales::{Locale, LocalizedAttributesRuleView}; +use meilisearch_types::milli::update::Setting; +use meilisearch_types::milli::vector::settings::EmbeddingSettings; +use meilisearch_types::settings::{ + FacetingSettings, PaginationSettings, ProximityPrecisionView, RankingRuleView, TypoSettings, +}; +use serde::Serialize; + +use crate::analytics::Aggregate; + +#[derive(Serialize, Default)] +pub struct SettingsAnalytics { + pub ranking_rules: RankingRulesAnalytics, + pub searchable_attributes: SearchableAttributesAnalytics, + pub displayed_attributes: DisplayedAttributesAnalytics, + pub sortable_attributes: SortableAttributesAnalytics, + pub filterable_attributes: FilterableAttributesAnalytics, + pub distinct_attribute: DistinctAttributeAnalytics, + pub proximity_precision: ProximityPrecisionAnalytics, + pub typo_tolerance: TypoToleranceAnalytics, + pub faceting: FacetingAnalytics, + pub pagination: PaginationAnalytics, + pub stop_words: StopWordsAnalytics, + pub synonyms: SynonymsAnalytics, + pub embedders: EmbeddersAnalytics, + pub search_cutoff_ms: SearchCutoffMsAnalytics, + pub locales: LocalesAnalytics, + pub dictionary: DictionaryAnalytics, + pub separator_tokens: SeparatorTokensAnalytics, + pub non_separator_tokens: NonSeparatorTokensAnalytics, +} + +impl Aggregate for SettingsAnalytics { + fn event_name(&self) -> &'static str { + "Settings Updated" + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { + ranking_rules: RankingRulesAnalytics { + words_position: new + .ranking_rules + .words_position + .or(self.ranking_rules.words_position), + typo_position: new.ranking_rules.typo_position.or(self.ranking_rules.typo_position), + proximity_position: new + .ranking_rules + .proximity_position + .or(self.ranking_rules.proximity_position), + attribute_position: new + .ranking_rules + .attribute_position + .or(self.ranking_rules.attribute_position), + sort_position: new.ranking_rules.sort_position.or(self.ranking_rules.sort_position), + exactness_position: new + .ranking_rules + .exactness_position + .or(self.ranking_rules.exactness_position), + values: new.ranking_rules.values.or(self.ranking_rules.values), + }, + searchable_attributes: SearchableAttributesAnalytics { + total: new.searchable_attributes.total.or(self.searchable_attributes.total), + with_wildcard: new + .searchable_attributes + .with_wildcard + .or(self.searchable_attributes.with_wildcard), + }, + displayed_attributes: DisplayedAttributesAnalytics { + total: new.displayed_attributes.total.or(self.displayed_attributes.total), + with_wildcard: new + .displayed_attributes + .with_wildcard + .or(self.displayed_attributes.with_wildcard), + }, + sortable_attributes: SortableAttributesAnalytics { + total: new.sortable_attributes.total.or(self.sortable_attributes.total), + has_geo: new.sortable_attributes.has_geo.or(self.sortable_attributes.has_geo), + }, + filterable_attributes: FilterableAttributesAnalytics { + total: new.filterable_attributes.total.or(self.filterable_attributes.total), + has_geo: new.filterable_attributes.has_geo.or(self.filterable_attributes.has_geo), + }, + distinct_attribute: DistinctAttributeAnalytics { + set: self.distinct_attribute.set | new.distinct_attribute.set, + }, + proximity_precision: ProximityPrecisionAnalytics { + set: self.proximity_precision.set | new.proximity_precision.set, + value: new.proximity_precision.value.or(self.proximity_precision.value), + }, + typo_tolerance: TypoToleranceAnalytics { + enabled: new.typo_tolerance.enabled.or(self.typo_tolerance.enabled), + disable_on_attributes: new + .typo_tolerance + .disable_on_attributes + .or(self.typo_tolerance.disable_on_attributes), + disable_on_words: new + .typo_tolerance + .disable_on_words + .or(self.typo_tolerance.disable_on_words), + min_word_size_for_one_typo: new + .typo_tolerance + .min_word_size_for_one_typo + .or(self.typo_tolerance.min_word_size_for_one_typo), + min_word_size_for_two_typos: new + .typo_tolerance + .min_word_size_for_two_typos + .or(self.typo_tolerance.min_word_size_for_two_typos), + }, + faceting: FacetingAnalytics { + max_values_per_facet: new + .faceting + .max_values_per_facet + .or(self.faceting.max_values_per_facet), + sort_facet_values_by_star_count: new + .faceting + .sort_facet_values_by_star_count + .or(self.faceting.sort_facet_values_by_star_count), + sort_facet_values_by_total: new + .faceting + .sort_facet_values_by_total + .or(self.faceting.sort_facet_values_by_total), + }, + pagination: PaginationAnalytics { + max_total_hits: new.pagination.max_total_hits.or(self.pagination.max_total_hits), + }, + stop_words: StopWordsAnalytics { + total: new.stop_words.total.or(self.stop_words.total), + }, + synonyms: SynonymsAnalytics { total: new.synonyms.total.or(self.synonyms.total) }, + embedders: EmbeddersAnalytics { + total: new.embedders.total.or(self.embedders.total), + sources: match (self.embedders.sources, new.embedders.sources) { + (None, None) => None, + (Some(sources), None) | (None, Some(sources)) => Some(sources), + (Some(this), Some(other)) => Some(this.union(&other).cloned().collect()), + }, + document_template_used: match ( + self.embedders.document_template_used, + new.embedders.document_template_used, + ) { + (None, None) => None, + (Some(used), None) | (None, Some(used)) => Some(used), + (Some(this), Some(other)) => Some(this | other), + }, + document_template_max_bytes: match ( + self.embedders.document_template_max_bytes, + new.embedders.document_template_max_bytes, + ) { + (None, None) => None, + (Some(bytes), None) | (None, Some(bytes)) => Some(bytes), + (Some(this), Some(other)) => Some(this.max(other)), + }, + binary_quantization_used: match ( + self.embedders.binary_quantization_used, + new.embedders.binary_quantization_used, + ) { + (None, None) => None, + (Some(bq), None) | (None, Some(bq)) => Some(bq), + (Some(this), Some(other)) => Some(this | other), + }, + }, + search_cutoff_ms: SearchCutoffMsAnalytics { + search_cutoff_ms: new + .search_cutoff_ms + .search_cutoff_ms + .or(self.search_cutoff_ms.search_cutoff_ms), + }, + locales: LocalesAnalytics { locales: new.locales.locales.or(self.locales.locales) }, + dictionary: DictionaryAnalytics { + total: new.dictionary.total.or(self.dictionary.total), + }, + separator_tokens: SeparatorTokensAnalytics { + total: new.non_separator_tokens.total.or(self.separator_tokens.total), + }, + non_separator_tokens: NonSeparatorTokensAnalytics { + total: new.non_separator_tokens.total.or(self.non_separator_tokens.total), + }, + }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + +#[derive(Serialize, Default)] +pub struct RankingRulesAnalytics { + pub words_position: Option, + pub typo_position: Option, + pub proximity_position: Option, + pub attribute_position: Option, + pub sort_position: Option, + pub exactness_position: Option, + pub values: Option, +} + +impl RankingRulesAnalytics { + pub fn new(rr: Option<&Vec>) -> Self { + RankingRulesAnalytics { + words_position: rr.as_ref().and_then(|rr| { + rr.iter() + .position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Words)) + }), + typo_position: rr.as_ref().and_then(|rr| { + rr.iter() + .position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Typo)) + }), + proximity_position: rr.as_ref().and_then(|rr| { + rr.iter().position(|s| { + matches!(s, meilisearch_types::settings::RankingRuleView::Proximity) + }) + }), + attribute_position: rr.as_ref().and_then(|rr| { + rr.iter().position(|s| { + matches!(s, meilisearch_types::settings::RankingRuleView::Attribute) + }) + }), + sort_position: rr.as_ref().and_then(|rr| { + rr.iter() + .position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Sort)) + }), + exactness_position: rr.as_ref().and_then(|rr| { + rr.iter().position(|s| { + matches!(s, meilisearch_types::settings::RankingRuleView::Exactness) + }) + }), + values: rr.as_ref().map(|rr| { + rr.iter() + .filter(|s| { + matches!( + s, + meilisearch_types::settings::RankingRuleView::Asc(_) + | meilisearch_types::settings::RankingRuleView::Desc(_) + ) + }) + .map(|x| x.to_string()) + .collect::>() + .join(", ") + }), + } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { ranking_rules: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct SearchableAttributesAnalytics { + pub total: Option, + pub with_wildcard: Option, +} + +impl SearchableAttributesAnalytics { + pub fn new(setting: Option<&Vec>) -> Self { + Self { + total: setting.as_ref().map(|searchable| searchable.len()), + with_wildcard: setting + .as_ref() + .map(|searchable| searchable.iter().any(|searchable| searchable == "*")), + } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { searchable_attributes: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct DisplayedAttributesAnalytics { + pub total: Option, + pub with_wildcard: Option, +} + +impl DisplayedAttributesAnalytics { + pub fn new(displayed: Option<&Vec>) -> Self { + Self { + total: displayed.as_ref().map(|displayed| displayed.len()), + with_wildcard: displayed + .as_ref() + .map(|displayed| displayed.iter().any(|displayed| displayed == "*")), + } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { displayed_attributes: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct SortableAttributesAnalytics { + pub total: Option, + pub has_geo: Option, +} + +impl SortableAttributesAnalytics { + pub fn new(setting: Option<&BTreeSet>) -> Self { + Self { + total: setting.as_ref().map(|sort| sort.len()), + has_geo: setting.as_ref().map(|sort| sort.contains("_geo")), + } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { sortable_attributes: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct FilterableAttributesAnalytics { + pub total: Option, + pub has_geo: Option, +} + +impl FilterableAttributesAnalytics { + pub fn new(setting: Option<&BTreeSet>) -> Self { + Self { + total: setting.as_ref().map(|filter| filter.len()), + has_geo: setting.as_ref().map(|filter| filter.contains("_geo")), + } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { filterable_attributes: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct DistinctAttributeAnalytics { + pub set: bool, +} + +impl DistinctAttributeAnalytics { + pub fn new(distinct: Option<&String>) -> Self { + Self { set: distinct.is_some() } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { distinct_attribute: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct ProximityPrecisionAnalytics { + pub set: bool, + pub value: Option, +} + +impl ProximityPrecisionAnalytics { + pub fn new(precision: Option<&ProximityPrecisionView>) -> Self { + Self { set: precision.is_some(), value: precision.cloned() } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { proximity_precision: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct TypoToleranceAnalytics { + pub enabled: Option, + pub disable_on_attributes: Option, + pub disable_on_words: Option, + pub min_word_size_for_one_typo: Option, + pub min_word_size_for_two_typos: Option, +} + +impl TypoToleranceAnalytics { + pub fn new(setting: Option<&TypoSettings>) -> Self { + Self { + enabled: setting.as_ref().map(|s| !matches!(s.enabled, Setting::Set(false))), + disable_on_attributes: setting + .as_ref() + .and_then(|s| s.disable_on_attributes.as_ref().set().map(|m| !m.is_empty())), + disable_on_words: setting + .as_ref() + .and_then(|s| s.disable_on_words.as_ref().set().map(|m| !m.is_empty())), + min_word_size_for_one_typo: setting + .as_ref() + .and_then(|s| s.min_word_size_for_typos.as_ref().set().map(|s| s.one_typo.set())) + .flatten(), + min_word_size_for_two_typos: setting + .as_ref() + .and_then(|s| s.min_word_size_for_typos.as_ref().set().map(|s| s.two_typos.set())) + .flatten(), + } + } + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { typo_tolerance: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct FacetingAnalytics { + pub max_values_per_facet: Option, + pub sort_facet_values_by_star_count: Option, + pub sort_facet_values_by_total: Option, +} + +impl FacetingAnalytics { + pub fn new(setting: Option<&FacetingSettings>) -> Self { + Self { + max_values_per_facet: setting.as_ref().and_then(|s| s.max_values_per_facet.set()), + sort_facet_values_by_star_count: setting.as_ref().and_then(|s| { + s.sort_facet_values_by + .as_ref() + .set() + .map(|s| s.iter().any(|(k, v)| k == "*" && v == &FacetValuesSort::Count)) + }), + sort_facet_values_by_total: setting + .as_ref() + .and_then(|s| s.sort_facet_values_by.as_ref().set().map(|s| s.len())), + } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { faceting: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct PaginationAnalytics { + pub max_total_hits: Option, +} + +impl PaginationAnalytics { + pub fn new(setting: Option<&PaginationSettings>) -> Self { + Self { max_total_hits: setting.as_ref().and_then(|s| s.max_total_hits.set()) } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { pagination: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct StopWordsAnalytics { + pub total: Option, +} + +impl StopWordsAnalytics { + pub fn new(stop_words: Option<&BTreeSet>) -> Self { + Self { total: stop_words.as_ref().map(|stop_words| stop_words.len()) } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { stop_words: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct SynonymsAnalytics { + pub total: Option, +} + +impl SynonymsAnalytics { + pub fn new(synonyms: Option<&BTreeMap>>) -> Self { + Self { total: synonyms.as_ref().map(|synonyms| synonyms.len()) } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { synonyms: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct EmbeddersAnalytics { + // last + pub total: Option, + // Merge the sources + pub sources: Option>, + // |= + pub document_template_used: Option, + // max + pub document_template_max_bytes: Option, + // |= + pub binary_quantization_used: Option, +} + +impl EmbeddersAnalytics { + pub fn new(setting: Option<&BTreeMap>>) -> Self { + let mut sources = std::collections::HashSet::new(); + + if let Some(s) = &setting { + for source in s + .values() + .filter_map(|config| config.clone().set()) + .filter_map(|config| config.source.set()) + { + use meilisearch_types::milli::vector::settings::EmbedderSource; + match source { + EmbedderSource::OpenAi => sources.insert("openAi".to_string()), + EmbedderSource::HuggingFace => sources.insert("huggingFace".to_string()), + EmbedderSource::UserProvided => sources.insert("userProvided".to_string()), + EmbedderSource::Ollama => sources.insert("ollama".to_string()), + EmbedderSource::Rest => sources.insert("rest".to_string()), + }; + } + }; + + Self { + total: setting.as_ref().map(|s| s.len()), + sources: Some(sources), + document_template_used: setting.as_ref().map(|map| { + map.values() + .filter_map(|config| config.clone().set()) + .any(|config| config.document_template.set().is_some()) + }), + document_template_max_bytes: setting.as_ref().and_then(|map| { + map.values() + .filter_map(|config| config.clone().set()) + .filter_map(|config| config.document_template_max_bytes.set()) + .max() + }), + binary_quantization_used: setting.as_ref().map(|map| { + map.values() + .filter_map(|config| config.clone().set()) + .any(|config| config.binary_quantized.set().is_some()) + }), + } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { embedders: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +#[serde(transparent)] +pub struct SearchCutoffMsAnalytics { + pub search_cutoff_ms: Option, +} + +impl SearchCutoffMsAnalytics { + pub fn new(setting: Option<&u64>) -> Self { + Self { search_cutoff_ms: setting.copied() } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { search_cutoff_ms: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +#[serde(transparent)] +pub struct LocalesAnalytics { + pub locales: Option>, +} + +impl LocalesAnalytics { + pub fn new(rules: Option<&Vec>) -> Self { + LocalesAnalytics { + locales: rules.as_ref().map(|rules| { + rules + .iter() + .flat_map(|rule| rule.locales.iter().cloned()) + .collect::>() + }), + } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { locales: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct DictionaryAnalytics { + pub total: Option, +} + +impl DictionaryAnalytics { + pub fn new(dictionary: Option<&BTreeSet>) -> Self { + Self { total: dictionary.as_ref().map(|dictionary| dictionary.len()) } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { dictionary: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct SeparatorTokensAnalytics { + pub total: Option, +} + +impl SeparatorTokensAnalytics { + pub fn new(separator_tokens: Option<&BTreeSet>) -> Self { + Self { total: separator_tokens.as_ref().map(|separator_tokens| separator_tokens.len()) } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { separator_tokens: self, ..Default::default() } + } +} + +#[derive(Serialize, Default)] +pub struct NonSeparatorTokensAnalytics { + pub total: Option, +} + +impl NonSeparatorTokensAnalytics { + pub fn new(non_separator_tokens: Option<&BTreeSet>) -> Self { + Self { + total: non_separator_tokens + .as_ref() + .map(|non_separator_tokens| non_separator_tokens.len()), + } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { non_separator_tokens: self, ..Default::default() } + } +} diff --git a/meilisearch/src/routes/indexes/similar.rs b/crates/meilisearch/src/routes/indexes/similar.rs similarity index 94% rename from meilisearch/src/routes/indexes/similar.rs rename to crates/meilisearch/src/routes/indexes/similar.rs index f94a02987..79f42f0aa 100644 --- a/meilisearch/src/routes/indexes/similar.rs +++ b/crates/meilisearch/src/routes/indexes/similar.rs @@ -13,9 +13,10 @@ use serde_json::Value; use tracing::debug; use super::ActionPolicy; -use crate::analytics::{Analytics, SimilarAggregator}; +use crate::analytics::Analytics; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; +use crate::routes::indexes::similar_analytics::{SimilarAggregator, SimilarGET, SimilarPOST}; use crate::search::{ add_search_rules, perform_similar, RankingScoreThresholdSimilar, RetrieveVectors, SearchKind, SimilarQuery, SimilarResult, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, @@ -34,13 +35,13 @@ pub async fn similar_get( index_uid: web::Path, params: AwebQueryParameter, req: HttpRequest, - analytics: web::Data, + analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; let query = params.0.try_into()?; - let mut aggregate = SimilarAggregator::from_query(&query, &req); + let mut aggregate = SimilarAggregator::::from_query(&query); debug!(parameters = ?query, "Similar get"); @@ -49,7 +50,7 @@ pub async fn similar_get( if let Ok(similar) = &similar { aggregate.succeed(similar); } - analytics.get_similar(aggregate); + analytics.publish(aggregate, &req); let similar = similar?; @@ -62,21 +63,21 @@ pub async fn similar_post( index_uid: web::Path, params: AwebJson, req: HttpRequest, - analytics: web::Data, + analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; let query = params.into_inner(); debug!(parameters = ?query, "Similar post"); - let mut aggregate = SimilarAggregator::from_query(&query, &req); + let mut aggregate = SimilarAggregator::::from_query(&query); let similar = similar(index_scheduler, index_uid, query).await; if let Ok(similar) = &similar { aggregate.succeed(similar); } - analytics.post_similar(aggregate); + analytics.publish(aggregate, &req); let similar = similar?; diff --git a/crates/meilisearch/src/routes/indexes/similar_analytics.rs b/crates/meilisearch/src/routes/indexes/similar_analytics.rs new file mode 100644 index 000000000..726839c3a --- /dev/null +++ b/crates/meilisearch/src/routes/indexes/similar_analytics.rs @@ -0,0 +1,233 @@ +use std::collections::{BinaryHeap, HashMap}; + +use once_cell::sync::Lazy; +use regex::Regex; +use serde_json::{json, Value}; + +use crate::aggregate_methods; +use crate::analytics::{Aggregate, AggregateMethod}; +use crate::search::{SimilarQuery, SimilarResult}; + +aggregate_methods!( + SimilarPOST => "Similar POST", + SimilarGET => "Similar GET", +); + +#[derive(Default)] +pub struct SimilarAggregator { + // requests + total_received: usize, + total_succeeded: usize, + time_spent: BinaryHeap, + + // filter + filter_with_geo_radius: bool, + filter_with_geo_bounding_box: bool, + // every time a request has a filter, this field must be incremented by the number of terms it contains + filter_sum_of_criteria_terms: usize, + // every time a request has a filter, this field must be incremented by one + filter_total_number_of_criteria: usize, + used_syntax: HashMap, + + // Whether a non-default embedder was specified + retrieve_vectors: bool, + + // pagination + max_limit: usize, + max_offset: usize, + + // formatting + max_attributes_to_retrieve: usize, + + // scoring + show_ranking_score: bool, + show_ranking_score_details: bool, + ranking_score_threshold: bool, + + marker: std::marker::PhantomData, +} + +impl SimilarAggregator { + #[allow(clippy::field_reassign_with_default)] + pub fn from_query(query: &SimilarQuery) -> Self { + let SimilarQuery { + id: _, + embedder: _, + offset, + limit, + attributes_to_retrieve: _, + retrieve_vectors, + show_ranking_score, + show_ranking_score_details, + filter, + ranking_score_threshold, + } = query; + + let mut ret = Self::default(); + + ret.total_received = 1; + + if let Some(ref filter) = filter { + static RE: Lazy = Lazy::new(|| Regex::new("AND | OR").unwrap()); + ret.filter_total_number_of_criteria = 1; + + let syntax = match filter { + Value::String(_) => "string".to_string(), + Value::Array(values) => { + if values.iter().map(|v| v.to_string()).any(|s| RE.is_match(&s)) { + "mixed".to_string() + } else { + "array".to_string() + } + } + _ => "none".to_string(), + }; + // convert the string to a HashMap + ret.used_syntax.insert(syntax, 1); + + let stringified_filters = filter.to_string(); + ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius("); + ret.filter_with_geo_bounding_box = stringified_filters.contains("_geoBoundingBox("); + ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count(); + } + + ret.max_limit = *limit; + ret.max_offset = *offset; + + ret.show_ranking_score = *show_ranking_score; + ret.show_ranking_score_details = *show_ranking_score_details; + ret.ranking_score_threshold = ranking_score_threshold.is_some(); + + ret.retrieve_vectors = *retrieve_vectors; + + ret + } + + pub fn succeed(&mut self, result: &SimilarResult) { + let SimilarResult { id: _, hits: _, processing_time_ms, hits_info: _ } = result; + + self.total_succeeded = self.total_succeeded.saturating_add(1); + + self.time_spent.push(*processing_time_ms as usize); + } +} + +impl Aggregate for SimilarAggregator { + fn event_name(&self) -> &'static str { + Method::event_name() + } + + /// Aggregate one [SimilarAggregator] into another. + fn aggregate(mut self: Box, new: Box) -> Box { + let Self { + total_received, + total_succeeded, + mut time_spent, + filter_with_geo_radius, + filter_with_geo_bounding_box, + filter_sum_of_criteria_terms, + filter_total_number_of_criteria, + used_syntax, + max_limit, + max_offset, + max_attributes_to_retrieve, + show_ranking_score, + show_ranking_score_details, + ranking_score_threshold, + retrieve_vectors, + marker: _, + } = *new; + + // request + self.total_received = self.total_received.saturating_add(total_received); + self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded); + self.time_spent.append(&mut time_spent); + + // filter + self.filter_with_geo_radius |= filter_with_geo_radius; + self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box; + self.filter_sum_of_criteria_terms = + self.filter_sum_of_criteria_terms.saturating_add(filter_sum_of_criteria_terms); + self.filter_total_number_of_criteria = + self.filter_total_number_of_criteria.saturating_add(filter_total_number_of_criteria); + for (key, value) in used_syntax.into_iter() { + let used_syntax = self.used_syntax.entry(key).or_insert(0); + *used_syntax = used_syntax.saturating_add(value); + } + + self.retrieve_vectors |= retrieve_vectors; + + // pagination + self.max_limit = self.max_limit.max(max_limit); + self.max_offset = self.max_offset.max(max_offset); + + // formatting + self.max_attributes_to_retrieve = + self.max_attributes_to_retrieve.max(max_attributes_to_retrieve); + + // scoring + self.show_ranking_score |= show_ranking_score; + self.show_ranking_score_details |= show_ranking_score_details; + self.ranking_score_threshold |= ranking_score_threshold; + + self + } + + fn into_event(self: Box) -> serde_json::Value { + let Self { + total_received, + total_succeeded, + time_spent, + filter_with_geo_radius, + filter_with_geo_bounding_box, + filter_sum_of_criteria_terms, + filter_total_number_of_criteria, + used_syntax, + max_limit, + max_offset, + max_attributes_to_retrieve, + show_ranking_score, + show_ranking_score_details, + ranking_score_threshold, + retrieve_vectors, + marker: _, + } = *self; + + // we get all the values in a sorted manner + let time_spent = time_spent.into_sorted_vec(); + // the index of the 99th percentage of value + let percentile_99th = time_spent.len() * 99 / 100; + // We are only interested by the slowest value of the 99th fastest results + let time_spent = time_spent.get(percentile_99th); + + json!({ + "requests": { + "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), + "total_succeeded": total_succeeded, + "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics + "total_received": total_received, + }, + "filter": { + "with_geoRadius": filter_with_geo_radius, + "with_geoBoundingBox": filter_with_geo_bounding_box, + "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64), + "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), + }, + "vector": { + "retrieve_vectors": retrieve_vectors, + }, + "pagination": { + "max_limit": max_limit, + "max_offset": max_offset, + }, + "formatting": { + "max_attributes_to_retrieve": max_attributes_to_retrieve, + }, + "scoring": { + "show_ranking_score": show_ranking_score, + "show_ranking_score_details": show_ranking_score_details, + "ranking_score_threshold": ranking_score_threshold, + } + }) + } +} diff --git a/meilisearch/src/routes/logs.rs b/crates/meilisearch/src/routes/logs.rs similarity index 100% rename from meilisearch/src/routes/logs.rs rename to crates/meilisearch/src/routes/logs.rs diff --git a/meilisearch/src/routes/metrics.rs b/crates/meilisearch/src/routes/metrics.rs similarity index 100% rename from meilisearch/src/routes/metrics.rs rename to crates/meilisearch/src/routes/metrics.rs diff --git a/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs similarity index 99% rename from meilisearch/src/routes/mod.rs rename to crates/meilisearch/src/routes/mod.rs index c25aeee70..b7260ea08 100644 --- a/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -25,6 +25,7 @@ pub mod indexes; mod logs; mod metrics; mod multi_search; +mod multi_search_analytics; mod snapshot; mod swap_indexes; pub mod tasks; diff --git a/meilisearch/src/routes/multi_search.rs b/crates/meilisearch/src/routes/multi_search.rs similarity index 96% rename from meilisearch/src/routes/multi_search.rs rename to crates/meilisearch/src/routes/multi_search.rs index 5fcb868c6..f8b1bc6ee 100644 --- a/meilisearch/src/routes/multi_search.rs +++ b/crates/meilisearch/src/routes/multi_search.rs @@ -9,7 +9,8 @@ use meilisearch_types::keys::actions; use serde::Serialize; use tracing::debug; -use crate::analytics::{Analytics, MultiSearchAggregator}; +use super::multi_search_analytics::MultiSearchAggregator; +use crate::analytics::Analytics; use crate::error::MeilisearchHttpError; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::{AuthenticationError, GuardedData}; @@ -35,7 +36,7 @@ pub async fn multi_search_with_post( search_queue: Data, params: AwebJson, req: HttpRequest, - analytics: web::Data, + analytics: web::Data, ) -> Result { // Since we don't want to process half of the search requests and then get a permit refused // we're going to get one permit for the whole duration of the multi-search request. @@ -43,7 +44,7 @@ pub async fn multi_search_with_post( let federated_search = params.into_inner(); - let mut multi_aggregate = MultiSearchAggregator::from_federated_search(&federated_search, &req); + let mut multi_aggregate = MultiSearchAggregator::from_federated_search(&federated_search); let FederatedSearch { mut queries, federation } = federated_search; @@ -87,7 +88,7 @@ pub async fn multi_search_with_post( multi_aggregate.succeed(); } - analytics.post_multi_search(multi_aggregate); + analytics.publish(multi_aggregate, &req); HttpResponse::Ok().json(search_result??) } None => { @@ -149,7 +150,7 @@ pub async fn multi_search_with_post( if search_results.is_ok() { multi_aggregate.succeed(); } - analytics.post_multi_search(multi_aggregate); + analytics.publish(multi_aggregate, &req); let search_results = search_results.map_err(|(mut err, query_index)| { // Add the query index that failed as context for the error message. diff --git a/crates/meilisearch/src/routes/multi_search_analytics.rs b/crates/meilisearch/src/routes/multi_search_analytics.rs new file mode 100644 index 000000000..3d07f471c --- /dev/null +++ b/crates/meilisearch/src/routes/multi_search_analytics.rs @@ -0,0 +1,168 @@ +use std::collections::HashSet; + +use serde_json::json; + +use crate::analytics::Aggregate; +use crate::search::{FederatedSearch, SearchQueryWithIndex}; + +#[derive(Default)] +pub struct MultiSearchAggregator { + // requests + total_received: usize, + total_succeeded: usize, + + // sum of the number of distinct indexes in each single request, use with total_received to compute an avg + total_distinct_index_count: usize, + // number of queries with a single index, use with total_received to compute a proportion + total_single_index: usize, + + // sum of the number of search queries in the requests, use with total_received to compute an average + total_search_count: usize, + + // scoring + show_ranking_score: bool, + show_ranking_score_details: bool, + + // federation + use_federation: bool, +} + +impl MultiSearchAggregator { + pub fn from_federated_search(federated_search: &FederatedSearch) -> Self { + let use_federation = federated_search.federation.is_some(); + + let distinct_indexes: HashSet<_> = federated_search + .queries + .iter() + .map(|query| { + let query = &query; + // make sure we get a compilation error if a field gets added to / removed from SearchQueryWithIndex + let SearchQueryWithIndex { + index_uid, + federation_options: _, + q: _, + vector: _, + offset: _, + limit: _, + page: _, + hits_per_page: _, + attributes_to_retrieve: _, + retrieve_vectors: _, + attributes_to_crop: _, + crop_length: _, + attributes_to_highlight: _, + show_ranking_score: _, + show_ranking_score_details: _, + show_matches_position: _, + filter: _, + sort: _, + distinct: _, + facets: _, + highlight_pre_tag: _, + highlight_post_tag: _, + crop_marker: _, + matching_strategy: _, + attributes_to_search_on: _, + hybrid: _, + ranking_score_threshold: _, + locales: _, + } = query; + + index_uid.as_str() + }) + .collect(); + + let show_ranking_score = + federated_search.queries.iter().any(|query| query.show_ranking_score); + let show_ranking_score_details = + federated_search.queries.iter().any(|query| query.show_ranking_score_details); + + Self { + total_received: 1, + total_succeeded: 0, + total_distinct_index_count: distinct_indexes.len(), + total_single_index: if distinct_indexes.len() == 1 { 1 } else { 0 }, + total_search_count: federated_search.queries.len(), + show_ranking_score, + show_ranking_score_details, + use_federation, + } + } + + pub fn succeed(&mut self) { + self.total_succeeded = self.total_succeeded.saturating_add(1); + } +} + +impl Aggregate for MultiSearchAggregator { + fn event_name(&self) -> &'static str { + "Documents Searched by Multi-Search POST" + } + + /// Aggregate one [MultiSearchAggregator] into another. + fn aggregate(self: Box, new: Box) -> Box { + // write the aggregate in a way that will cause a compilation error if a field is added. + + // get ownership of self, replacing it by a default value. + let this = *self; + + let total_received = this.total_received.saturating_add(new.total_received); + let total_succeeded = this.total_succeeded.saturating_add(new.total_succeeded); + let total_distinct_index_count = + this.total_distinct_index_count.saturating_add(new.total_distinct_index_count); + let total_single_index = this.total_single_index.saturating_add(new.total_single_index); + let total_search_count = this.total_search_count.saturating_add(new.total_search_count); + let show_ranking_score = this.show_ranking_score || new.show_ranking_score; + let show_ranking_score_details = + this.show_ranking_score_details || new.show_ranking_score_details; + let use_federation = this.use_federation || new.use_federation; + + Box::new(Self { + total_received, + total_succeeded, + total_distinct_index_count, + total_single_index, + total_search_count, + show_ranking_score, + show_ranking_score_details, + use_federation, + }) + } + + fn into_event(self: Box) -> serde_json::Value { + let Self { + total_received, + total_succeeded, + total_distinct_index_count, + total_single_index, + total_search_count, + show_ranking_score, + show_ranking_score_details, + use_federation, + } = *self; + + json!({ + "requests": { + "total_succeeded": total_succeeded, + "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics + "total_received": total_received, + }, + "indexes": { + "total_single_index": total_single_index, + "total_distinct_index_count": total_distinct_index_count, + "avg_distinct_index_count": (total_distinct_index_count as f64) / (total_received as f64), // not 0 else returned early + }, + "searches": { + "total_search_count": total_search_count, + "avg_search_count": (total_search_count as f64) / (total_received as f64), + }, + "scoring": { + "show_ranking_score": show_ranking_score, + "show_ranking_score_details": show_ranking_score_details, + }, + "federation": { + "use_federation": use_federation, + } + }) + } +} diff --git a/meilisearch/src/routes/snapshot.rs b/crates/meilisearch/src/routes/snapshot.rs similarity index 88% rename from meilisearch/src/routes/snapshot.rs rename to crates/meilisearch/src/routes/snapshot.rs index 84673729f..cacbc41af 100644 --- a/meilisearch/src/routes/snapshot.rs +++ b/crates/meilisearch/src/routes/snapshot.rs @@ -3,7 +3,6 @@ use actix_web::{web, HttpRequest, HttpResponse}; use index_scheduler::IndexScheduler; use meilisearch_types::error::ResponseError; use meilisearch_types::tasks::KindWithContent; -use serde_json::json; use tracing::debug; use crate::analytics::Analytics; @@ -17,13 +16,15 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::resource("").route(web::post().to(SeqHandler(create_snapshot)))); } +crate::empty_analytics!(SnapshotAnalytics, "Snapshot Created"); + pub async fn create_snapshot( index_scheduler: GuardedData, Data>, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { - analytics.publish("Snapshot Created".to_string(), json!({}), Some(&req)); + analytics.publish(SnapshotAnalytics::default(), &req); let task = KindWithContent::SnapshotCreation; let uid = get_task_id(&req, &opt)?; diff --git a/meilisearch/src/routes/swap_indexes.rs b/crates/meilisearch/src/routes/swap_indexes.rs similarity index 77% rename from meilisearch/src/routes/swap_indexes.rs rename to crates/meilisearch/src/routes/swap_indexes.rs index 51a7b0707..9b8b67e63 100644 --- a/meilisearch/src/routes/swap_indexes.rs +++ b/crates/meilisearch/src/routes/swap_indexes.rs @@ -8,10 +8,10 @@ use meilisearch_types::error::deserr_codes::InvalidSwapIndexes; use meilisearch_types::error::ResponseError; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::tasks::{IndexSwap, KindWithContent}; -use serde_json::json; +use serde::Serialize; use super::{get_task_id, is_dry_run, SummarizedTaskView}; -use crate::analytics::Analytics; +use crate::analytics::{Aggregate, Analytics}; use crate::error::MeilisearchHttpError; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::{AuthenticationError, GuardedData}; @@ -29,21 +29,36 @@ pub struct SwapIndexesPayload { indexes: Vec, } +#[derive(Serialize)] +struct IndexSwappedAnalytics { + swap_operation_number: usize, +} + +impl Aggregate for IndexSwappedAnalytics { + fn event_name(&self) -> &'static str { + "Indexes Swapped" + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { + swap_operation_number: self.swap_operation_number.max(new.swap_operation_number), + }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + pub async fn swap_indexes( index_scheduler: GuardedData, Data>, params: AwebJson, DeserrJsonError>, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { let params = params.into_inner(); - analytics.publish( - "Indexes Swapped".to_string(), - json!({ - "swap_operation_number": params.len(), - }), - Some(&req), - ); + analytics.publish(IndexSwappedAnalytics { swap_operation_number: params.len() }, &req); let filters = index_scheduler.filters(); let mut swaps = vec![]; diff --git a/meilisearch/src/routes/tasks.rs b/crates/meilisearch/src/routes/tasks.rs similarity index 88% rename from meilisearch/src/routes/tasks.rs rename to crates/meilisearch/src/routes/tasks.rs index 3dc6520af..95959d6d5 100644 --- a/meilisearch/src/routes/tasks.rs +++ b/crates/meilisearch/src/routes/tasks.rs @@ -12,18 +12,17 @@ use meilisearch_types::star_or::{OptionStarOr, OptionStarOrList}; use meilisearch_types::task_view::TaskView; use meilisearch_types::tasks::{Kind, KindWithContent, Status}; use serde::Serialize; -use serde_json::json; use time::format_description::well_known::Rfc3339; use time::macros::format_description; use time::{Date, Duration, OffsetDateTime, Time}; use tokio::task; use super::{get_task_id, is_dry_run, SummarizedTaskView}; -use crate::analytics::Analytics; +use crate::analytics::{Aggregate, AggregateMethod, Analytics}; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; -use crate::Opt; +use crate::{aggregate_methods, Opt}; const DEFAULT_LIMIT: u32 = 20; @@ -158,12 +157,69 @@ impl TaskDeletionOrCancelationQuery { } } +aggregate_methods!( + CancelTasks => "Tasks Canceled", + DeleteTasks => "Tasks Deleted", +); + +#[derive(Serialize)] +struct TaskFilterAnalytics { + filtered_by_uid: bool, + filtered_by_index_uid: bool, + filtered_by_type: bool, + filtered_by_status: bool, + filtered_by_canceled_by: bool, + filtered_by_before_enqueued_at: bool, + filtered_by_after_enqueued_at: bool, + filtered_by_before_started_at: bool, + filtered_by_after_started_at: bool, + filtered_by_before_finished_at: bool, + filtered_by_after_finished_at: bool, + + #[serde(skip)] + marker: std::marker::PhantomData, +} + +impl Aggregate for TaskFilterAnalytics { + fn event_name(&self) -> &'static str { + Method::event_name() + } + + fn aggregate(self: Box, new: Box) -> Box { + Box::new(Self { + filtered_by_uid: self.filtered_by_uid | new.filtered_by_uid, + filtered_by_index_uid: self.filtered_by_index_uid | new.filtered_by_index_uid, + filtered_by_type: self.filtered_by_type | new.filtered_by_type, + filtered_by_status: self.filtered_by_status | new.filtered_by_status, + filtered_by_canceled_by: self.filtered_by_canceled_by | new.filtered_by_canceled_by, + filtered_by_before_enqueued_at: self.filtered_by_before_enqueued_at + | new.filtered_by_before_enqueued_at, + filtered_by_after_enqueued_at: self.filtered_by_after_enqueued_at + | new.filtered_by_after_enqueued_at, + filtered_by_before_started_at: self.filtered_by_before_started_at + | new.filtered_by_before_started_at, + filtered_by_after_started_at: self.filtered_by_after_started_at + | new.filtered_by_after_started_at, + filtered_by_before_finished_at: self.filtered_by_before_finished_at + | new.filtered_by_before_finished_at, + filtered_by_after_finished_at: self.filtered_by_after_finished_at + | new.filtered_by_after_finished_at, + + marker: std::marker::PhantomData, + }) + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + async fn cancel_tasks( index_scheduler: GuardedData, Data>, params: AwebQueryParameter, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { let params = params.into_inner(); @@ -172,21 +228,22 @@ async fn cancel_tasks( } analytics.publish( - "Tasks Canceled".to_string(), - json!({ - "filtered_by_uid": params.uids.is_some(), - "filtered_by_index_uid": params.index_uids.is_some(), - "filtered_by_type": params.types.is_some(), - "filtered_by_status": params.statuses.is_some(), - "filtered_by_canceled_by": params.canceled_by.is_some(), - "filtered_by_before_enqueued_at": params.before_enqueued_at.is_some(), - "filtered_by_after_enqueued_at": params.after_enqueued_at.is_some(), - "filtered_by_before_started_at": params.before_started_at.is_some(), - "filtered_by_after_started_at": params.after_started_at.is_some(), - "filtered_by_before_finished_at": params.before_finished_at.is_some(), - "filtered_by_after_finished_at": params.after_finished_at.is_some(), - }), - Some(&req), + TaskFilterAnalytics:: { + filtered_by_uid: params.uids.is_some(), + filtered_by_index_uid: params.index_uids.is_some(), + filtered_by_type: params.types.is_some(), + filtered_by_status: params.statuses.is_some(), + filtered_by_canceled_by: params.canceled_by.is_some(), + filtered_by_before_enqueued_at: params.before_enqueued_at.is_some(), + filtered_by_after_enqueued_at: params.after_enqueued_at.is_some(), + filtered_by_before_started_at: params.before_started_at.is_some(), + filtered_by_after_started_at: params.after_started_at.is_some(), + filtered_by_before_finished_at: params.before_finished_at.is_some(), + filtered_by_after_finished_at: params.after_finished_at.is_some(), + + marker: std::marker::PhantomData, + }, + &req, ); let query = params.into_query(); @@ -214,7 +271,7 @@ async fn delete_tasks( params: AwebQueryParameter, req: HttpRequest, opt: web::Data, - analytics: web::Data, + analytics: web::Data, ) -> Result { let params = params.into_inner(); @@ -223,22 +280,24 @@ async fn delete_tasks( } analytics.publish( - "Tasks Deleted".to_string(), - json!({ - "filtered_by_uid": params.uids.is_some(), - "filtered_by_index_uid": params.index_uids.is_some(), - "filtered_by_type": params.types.is_some(), - "filtered_by_status": params.statuses.is_some(), - "filtered_by_canceled_by": params.canceled_by.is_some(), - "filtered_by_before_enqueued_at": params.before_enqueued_at.is_some(), - "filtered_by_after_enqueued_at": params.after_enqueued_at.is_some(), - "filtered_by_before_started_at": params.before_started_at.is_some(), - "filtered_by_after_started_at": params.after_started_at.is_some(), - "filtered_by_before_finished_at": params.before_finished_at.is_some(), - "filtered_by_after_finished_at": params.after_finished_at.is_some(), - }), - Some(&req), + TaskFilterAnalytics:: { + filtered_by_uid: params.uids.is_some(), + filtered_by_index_uid: params.index_uids.is_some(), + filtered_by_type: params.types.is_some(), + filtered_by_status: params.statuses.is_some(), + filtered_by_canceled_by: params.canceled_by.is_some(), + filtered_by_before_enqueued_at: params.before_enqueued_at.is_some(), + filtered_by_after_enqueued_at: params.after_enqueued_at.is_some(), + filtered_by_before_started_at: params.before_started_at.is_some(), + filtered_by_after_started_at: params.after_started_at.is_some(), + filtered_by_before_finished_at: params.before_finished_at.is_some(), + filtered_by_after_finished_at: params.after_finished_at.is_some(), + + marker: std::marker::PhantomData, + }, + &req, ); + let query = params.into_query(); let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes( @@ -616,7 +675,7 @@ mod tests { let err = deserr_query_params::(params).unwrap_err(); snapshot!(meili_snap::json_string!(err), @r###" { - "message": "Invalid value in parameter `indexUids[1]`: `hé` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "Invalid value in parameter `indexUids[1]`: `hé` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -628,7 +687,7 @@ mod tests { let err = deserr_query_params::(params).unwrap_err(); snapshot!(meili_snap::json_string!(err), @r###" { - "message": "Invalid value in parameter `indexUids`: `hé` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "Invalid value in parameter `indexUids`: `hé` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" diff --git a/meilisearch/src/search/federated.rs b/crates/meilisearch/src/search/federated.rs similarity index 100% rename from meilisearch/src/search/federated.rs rename to crates/meilisearch/src/search/federated.rs diff --git a/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs similarity index 99% rename from meilisearch/src/search/mod.rs rename to crates/meilisearch/src/search/mod.rs index 4e25f3a1a..c873ab387 100644 --- a/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1195,8 +1195,13 @@ impl<'a> HitMaker<'a> { let vectors_is_hidden = match (&displayed_ids, vectors_fid) { // displayed_ids is a wildcard, so `_vectors` can be displayed regardless of its fid (None, _) => false, - // displayed_ids is a finite list, and `_vectors` cannot be part of it because it is not an existing field - (Some(_), None) => true, + // vectors has no fid, so check its explicit name + (Some(_), None) => { + // unwrap as otherwise we'd go to the first one + let displayed_names = index.displayed_fields(rtxn)?.unwrap(); + !displayed_names + .contains(&milli::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME) + } // displayed_ids is a finit list, so hide if `_vectors` is not part of it (Some(map), Some(vectors_fid)) => map.contains(&vectors_fid), }; diff --git a/meilisearch/src/search/ranking_rules.rs b/crates/meilisearch/src/search/ranking_rules.rs similarity index 100% rename from meilisearch/src/search/ranking_rules.rs rename to crates/meilisearch/src/search/ranking_rules.rs diff --git a/meilisearch/src/search_queue.rs b/crates/meilisearch/src/search_queue.rs similarity index 100% rename from meilisearch/src/search_queue.rs rename to crates/meilisearch/src/search_queue.rs diff --git a/meilisearch/tests/assets/dumps/v1/metadata.json b/crates/meilisearch/tests/assets/dumps/v1/metadata.json similarity index 100% rename from meilisearch/tests/assets/dumps/v1/metadata.json rename to crates/meilisearch/tests/assets/dumps/v1/metadata.json diff --git a/meilisearch/tests/assets/dumps/v1/test/documents.jsonl b/crates/meilisearch/tests/assets/dumps/v1/test/documents.jsonl similarity index 100% rename from meilisearch/tests/assets/dumps/v1/test/documents.jsonl rename to crates/meilisearch/tests/assets/dumps/v1/test/documents.jsonl diff --git a/meilisearch/tests/assets/dumps/v1/test/settings.json b/crates/meilisearch/tests/assets/dumps/v1/test/settings.json similarity index 100% rename from meilisearch/tests/assets/dumps/v1/test/settings.json rename to crates/meilisearch/tests/assets/dumps/v1/test/settings.json diff --git a/meilisearch/tests/assets/dumps/v1/test/updates.jsonl b/crates/meilisearch/tests/assets/dumps/v1/test/updates.jsonl similarity index 100% rename from meilisearch/tests/assets/dumps/v1/test/updates.jsonl rename to crates/meilisearch/tests/assets/dumps/v1/test/updates.jsonl diff --git a/meilisearch/tests/assets/test_set.json b/crates/meilisearch/tests/assets/test_set.json similarity index 100% rename from meilisearch/tests/assets/test_set.json rename to crates/meilisearch/tests/assets/test_set.json diff --git a/meilisearch/tests/assets/test_set.ndjson b/crates/meilisearch/tests/assets/test_set.ndjson similarity index 100% rename from meilisearch/tests/assets/test_set.ndjson rename to crates/meilisearch/tests/assets/test_set.ndjson diff --git a/meilisearch/tests/assets/v1_v0.20.0_movies.dump b/crates/meilisearch/tests/assets/v1_v0.20.0_movies.dump similarity index 100% rename from meilisearch/tests/assets/v1_v0.20.0_movies.dump rename to crates/meilisearch/tests/assets/v1_v0.20.0_movies.dump diff --git a/meilisearch/tests/assets/v1_v0.20.0_movies_with_settings.dump b/crates/meilisearch/tests/assets/v1_v0.20.0_movies_with_settings.dump similarity index 100% rename from meilisearch/tests/assets/v1_v0.20.0_movies_with_settings.dump rename to crates/meilisearch/tests/assets/v1_v0.20.0_movies_with_settings.dump diff --git a/meilisearch/tests/assets/v1_v0.20.0_rubygems_with_settings.dump b/crates/meilisearch/tests/assets/v1_v0.20.0_rubygems_with_settings.dump similarity index 100% rename from meilisearch/tests/assets/v1_v0.20.0_rubygems_with_settings.dump rename to crates/meilisearch/tests/assets/v1_v0.20.0_rubygems_with_settings.dump diff --git a/meilisearch/tests/assets/v2_v0.21.1_movies.dump b/crates/meilisearch/tests/assets/v2_v0.21.1_movies.dump similarity index 100% rename from meilisearch/tests/assets/v2_v0.21.1_movies.dump rename to crates/meilisearch/tests/assets/v2_v0.21.1_movies.dump diff --git a/meilisearch/tests/assets/v2_v0.21.1_movies_with_settings.dump b/crates/meilisearch/tests/assets/v2_v0.21.1_movies_with_settings.dump similarity index 100% rename from meilisearch/tests/assets/v2_v0.21.1_movies_with_settings.dump rename to crates/meilisearch/tests/assets/v2_v0.21.1_movies_with_settings.dump diff --git a/meilisearch/tests/assets/v2_v0.21.1_rubygems_with_settings.dump b/crates/meilisearch/tests/assets/v2_v0.21.1_rubygems_with_settings.dump similarity index 100% rename from meilisearch/tests/assets/v2_v0.21.1_rubygems_with_settings.dump rename to crates/meilisearch/tests/assets/v2_v0.21.1_rubygems_with_settings.dump diff --git a/meilisearch/tests/assets/v3_v0.24.0_movies.dump b/crates/meilisearch/tests/assets/v3_v0.24.0_movies.dump similarity index 100% rename from meilisearch/tests/assets/v3_v0.24.0_movies.dump rename to crates/meilisearch/tests/assets/v3_v0.24.0_movies.dump diff --git a/meilisearch/tests/assets/v3_v0.24.0_movies_with_settings.dump b/crates/meilisearch/tests/assets/v3_v0.24.0_movies_with_settings.dump similarity index 100% rename from meilisearch/tests/assets/v3_v0.24.0_movies_with_settings.dump rename to crates/meilisearch/tests/assets/v3_v0.24.0_movies_with_settings.dump diff --git a/meilisearch/tests/assets/v3_v0.24.0_rubygems_with_settings.dump b/crates/meilisearch/tests/assets/v3_v0.24.0_rubygems_with_settings.dump similarity index 100% rename from meilisearch/tests/assets/v3_v0.24.0_rubygems_with_settings.dump rename to crates/meilisearch/tests/assets/v3_v0.24.0_rubygems_with_settings.dump diff --git a/meilisearch/tests/assets/v4_v0.25.2_movies.dump b/crates/meilisearch/tests/assets/v4_v0.25.2_movies.dump similarity index 100% rename from meilisearch/tests/assets/v4_v0.25.2_movies.dump rename to crates/meilisearch/tests/assets/v4_v0.25.2_movies.dump diff --git a/meilisearch/tests/assets/v4_v0.25.2_movies_with_settings.dump b/crates/meilisearch/tests/assets/v4_v0.25.2_movies_with_settings.dump similarity index 100% rename from meilisearch/tests/assets/v4_v0.25.2_movies_with_settings.dump rename to crates/meilisearch/tests/assets/v4_v0.25.2_movies_with_settings.dump diff --git a/meilisearch/tests/assets/v4_v0.25.2_rubygems_with_settings.dump b/crates/meilisearch/tests/assets/v4_v0.25.2_rubygems_with_settings.dump similarity index 100% rename from meilisearch/tests/assets/v4_v0.25.2_rubygems_with_settings.dump rename to crates/meilisearch/tests/assets/v4_v0.25.2_rubygems_with_settings.dump diff --git a/meilisearch/tests/assets/v5_v0.28.0_test_dump.dump b/crates/meilisearch/tests/assets/v5_v0.28.0_test_dump.dump similarity index 100% rename from meilisearch/tests/assets/v5_v0.28.0_test_dump.dump rename to crates/meilisearch/tests/assets/v5_v0.28.0_test_dump.dump diff --git a/meilisearch/tests/assets/v6_v1.6.0_use_deactivated_experimental_setting.dump b/crates/meilisearch/tests/assets/v6_v1.6.0_use_deactivated_experimental_setting.dump similarity index 100% rename from meilisearch/tests/assets/v6_v1.6.0_use_deactivated_experimental_setting.dump rename to crates/meilisearch/tests/assets/v6_v1.6.0_use_deactivated_experimental_setting.dump diff --git a/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs similarity index 100% rename from meilisearch/tests/auth/api_keys.rs rename to crates/meilisearch/tests/auth/api_keys.rs diff --git a/meilisearch/tests/auth/authorization.rs b/crates/meilisearch/tests/auth/authorization.rs similarity index 100% rename from meilisearch/tests/auth/authorization.rs rename to crates/meilisearch/tests/auth/authorization.rs diff --git a/meilisearch/tests/auth/errors.rs b/crates/meilisearch/tests/auth/errors.rs similarity index 100% rename from meilisearch/tests/auth/errors.rs rename to crates/meilisearch/tests/auth/errors.rs diff --git a/meilisearch/tests/auth/mod.rs b/crates/meilisearch/tests/auth/mod.rs similarity index 100% rename from meilisearch/tests/auth/mod.rs rename to crates/meilisearch/tests/auth/mod.rs diff --git a/meilisearch/tests/auth/payload.rs b/crates/meilisearch/tests/auth/payload.rs similarity index 100% rename from meilisearch/tests/auth/payload.rs rename to crates/meilisearch/tests/auth/payload.rs diff --git a/meilisearch/tests/auth/tenant_token.rs b/crates/meilisearch/tests/auth/tenant_token.rs similarity index 100% rename from meilisearch/tests/auth/tenant_token.rs rename to crates/meilisearch/tests/auth/tenant_token.rs diff --git a/meilisearch/tests/auth/tenant_token_multi_search.rs b/crates/meilisearch/tests/auth/tenant_token_multi_search.rs similarity index 100% rename from meilisearch/tests/auth/tenant_token_multi_search.rs rename to crates/meilisearch/tests/auth/tenant_token_multi_search.rs diff --git a/meilisearch/tests/common/encoder.rs b/crates/meilisearch/tests/common/encoder.rs similarity index 100% rename from meilisearch/tests/common/encoder.rs rename to crates/meilisearch/tests/common/encoder.rs diff --git a/meilisearch/tests/common/index.rs b/crates/meilisearch/tests/common/index.rs similarity index 96% rename from meilisearch/tests/common/index.rs rename to crates/meilisearch/tests/common/index.rs index 381bd1cb4..221333fd7 100644 --- a/meilisearch/tests/common/index.rs +++ b/crates/meilisearch/tests/common/index.rs @@ -9,8 +9,7 @@ use urlencoding::encode as urlencode; use super::encoder::Encoder; use super::service::Service; -use super::Value; -use super::{Owned, Shared}; +use super::{Owned, Shared, Value}; use crate::json; pub struct Index<'a, State = Owned> { @@ -272,6 +271,20 @@ impl<'a> Index<'a, Shared> { } (task, code) } + + pub async fn delete_index_fail(&self) -> (Value, StatusCode) { + let (mut task, code) = self._delete().await; + if code.is_success() { + task = self.wait_task(task.uid()).await; + if task.is_success() { + panic!( + "`delete_index_fail` succeeded: {}", + serde_json::to_string_pretty(&task).unwrap() + ); + } + } + (task, code) + } } #[allow(dead_code)] @@ -314,6 +327,12 @@ impl Index<'_, State> { }); self.service.post_encoded("/indexes", body, self.encoder).await } + + pub(super) async fn _delete(&self) -> (Value, StatusCode) { + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); + self.service.delete(url).await + } + pub async fn wait_task(&self, update_id: u64) -> Value { // try several times to get status, or panic to not wait forever let url = format!("/tasks/{}", update_id); diff --git a/meilisearch/tests/common/mod.rs b/crates/meilisearch/tests/common/mod.rs similarity index 100% rename from meilisearch/tests/common/mod.rs rename to crates/meilisearch/tests/common/mod.rs diff --git a/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs similarity index 98% rename from meilisearch/tests/common/server.rs rename to crates/meilisearch/tests/common/server.rs index 6d331ebbc..5069c9ea6 100644 --- a/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -309,6 +309,11 @@ impl Server { } } + pub fn unique_index_with_encoder(&self, encoder: Encoder) -> Index<'_> { + let uuid = Uuid::new_v4(); + Index { uid: uuid.to_string(), service: &self.service, encoder, marker: PhantomData } + } + pub(super) async fn _create_index(&self, body: Value) -> (Value, StatusCode) { self.service.post("/indexes", body).await } @@ -381,7 +386,6 @@ pub fn default_settings(dir: impl AsRef) -> Opt { db_path: dir.as_ref().join("db"), dump_dir: dir.as_ref().join("dumps"), env: "development".to_owned(), - #[cfg(feature = "analytics")] no_analytics: true, max_index_size: Byte::from_u64_with_unit(100, Unit::MiB).unwrap(), max_task_db_size: Byte::from_u64_with_unit(1, Unit::GiB).unwrap(), diff --git a/meilisearch/tests/common/service.rs b/crates/meilisearch/tests/common/service.rs similarity index 97% rename from meilisearch/tests/common/service.rs rename to crates/meilisearch/tests/common/service.rs index 8addbacf8..c0b07c217 100644 --- a/meilisearch/tests/common/service.rs +++ b/crates/meilisearch/tests/common/service.rs @@ -9,8 +9,9 @@ use actix_web::test; use actix_web::test::TestRequest; use actix_web::web::Data; use index_scheduler::IndexScheduler; +use meilisearch::analytics::Analytics; use meilisearch::search_queue::SearchQueue; -use meilisearch::{analytics, create_app, Opt, SubscriberForSecondLayer}; +use meilisearch::{create_app, Opt, SubscriberForSecondLayer}; use meilisearch_auth::AuthController; use tracing::level_filters::LevelFilter; use tracing_subscriber::Layer; @@ -141,7 +142,7 @@ impl Service { Data::new(search_queue), self.options.clone(), (route_layer_handle, stderr_layer_handle), - analytics::MockAnalytics::new(&self.options), + Data::new(Analytics::no_analytics()), true, )) .await diff --git a/meilisearch/tests/content_type.rs b/crates/meilisearch/tests/content_type.rs similarity index 100% rename from meilisearch/tests/content_type.rs rename to crates/meilisearch/tests/content_type.rs diff --git a/meilisearch/tests/dashboard/mod.rs b/crates/meilisearch/tests/dashboard/mod.rs similarity index 100% rename from meilisearch/tests/dashboard/mod.rs rename to crates/meilisearch/tests/dashboard/mod.rs diff --git a/meilisearch/tests/documents/add_documents.rs b/crates/meilisearch/tests/documents/add_documents.rs similarity index 99% rename from meilisearch/tests/documents/add_documents.rs rename to crates/meilisearch/tests/documents/add_documents.rs index 819b2ddc2..c37b3a5e3 100644 --- a/meilisearch/tests/documents/add_documents.rs +++ b/crates/meilisearch/tests/documents/add_documents.rs @@ -1023,7 +1023,7 @@ async fn error_document_add_create_index_bad_uid() { snapshot!(json_string!(response), @r###" { - "message": "`883 fj!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "`883 fj!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -1280,7 +1280,7 @@ async fn error_add_documents_bad_document_id() { "indexedDocuments": 0 }, "error": { - "message": "Document identifier `\"foo & bar\"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_).", + "message": "Document identifier `\"foo & bar\"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_document_id", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_document_id" diff --git a/meilisearch/tests/documents/delete_documents.rs b/crates/meilisearch/tests/documents/delete_documents.rs similarity index 100% rename from meilisearch/tests/documents/delete_documents.rs rename to crates/meilisearch/tests/documents/delete_documents.rs diff --git a/meilisearch/tests/documents/errors.rs b/crates/meilisearch/tests/documents/errors.rs similarity index 100% rename from meilisearch/tests/documents/errors.rs rename to crates/meilisearch/tests/documents/errors.rs diff --git a/meilisearch/tests/documents/get_documents.rs b/crates/meilisearch/tests/documents/get_documents.rs similarity index 100% rename from meilisearch/tests/documents/get_documents.rs rename to crates/meilisearch/tests/documents/get_documents.rs diff --git a/meilisearch/tests/documents/mod.rs b/crates/meilisearch/tests/documents/mod.rs similarity index 100% rename from meilisearch/tests/documents/mod.rs rename to crates/meilisearch/tests/documents/mod.rs diff --git a/meilisearch/tests/documents/update_documents.rs b/crates/meilisearch/tests/documents/update_documents.rs similarity index 76% rename from meilisearch/tests/documents/update_documents.rs rename to crates/meilisearch/tests/documents/update_documents.rs index a5d466513..c0703e81b 100644 --- a/meilisearch/tests/documents/update_documents.rs +++ b/crates/meilisearch/tests/documents/update_documents.rs @@ -11,7 +11,7 @@ async fn error_document_update_create_index_bad_uid() { let (response, code) = index.update_documents(json!([{"id": 1}]), None).await; let expected_response = json!({ - "message": "`883 fj!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "`883 fj!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -23,8 +23,8 @@ async fn error_document_update_create_index_bad_uid() { #[actix_rt::test] async fn document_update_with_primary_key() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let documents = json!([ { @@ -32,15 +32,14 @@ async fn document_update_with_primary_key() { "content": "foo", } ]); - let (_response, code) = index.update_documents(documents, Some("primary")).await; + let (response, code) = index.update_documents(documents, Some("primary")).await; assert_eq!(code, 202); - index.wait_task(0).await; + index.wait_task(response.uid()).await.succeeded(); - let (response, code) = index.get_task(0).await; + let (response, code) = index.get_task(response.uid()).await; assert_eq!(code, 200); assert_eq!(response["status"], "succeeded"); - assert_eq!(response["uid"], 0); assert_eq!(response["type"], "documentAdditionOrUpdate"); assert_eq!(response["details"]["indexedDocuments"], 1); assert_eq!(response["details"]["receivedDocuments"], 1); @@ -52,8 +51,8 @@ async fn document_update_with_primary_key() { #[actix_rt::test] async fn update_document() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let documents = json!([ { @@ -62,10 +61,10 @@ async fn update_document() { } ]); - let (_response, code) = index.add_documents(documents, None).await; + let (response, code) = index.add_documents(documents, None).await; assert_eq!(code, 202); - index.wait_task(0).await; + index.wait_task(response.uid()).await.succeeded(); let documents = json!([ { @@ -77,9 +76,9 @@ async fn update_document() { let (response, code) = index.update_documents(documents, None).await; assert_eq!(code, 202, "response: {}", response); - index.wait_task(1).await; + index.wait_task(response.uid()).await.succeeded(); - let (response, code) = index.get_task(1).await; + let (response, code) = index.get_task(response.uid()).await; assert_eq!(code, 200); assert_eq!(response["status"], "succeeded"); @@ -96,8 +95,8 @@ async fn update_document() { #[actix_rt::test] async fn update_document_gzip_encoded() { - let server = Server::new().await; - let index = server.index_with_encoder("test", Encoder::Gzip); + let server = Server::new_shared(); + let index = server.unique_index_with_encoder(Encoder::Gzip); let documents = json!([ { @@ -106,10 +105,10 @@ async fn update_document_gzip_encoded() { } ]); - let (_response, code) = index.add_documents(documents, None).await; + let (response, code) = index.add_documents(documents, None).await; assert_eq!(code, 202); - index.wait_task(0).await; + index.wait_task(response.uid()).await.succeeded(); let documents = json!([ { @@ -121,9 +120,9 @@ async fn update_document_gzip_encoded() { let (response, code) = index.update_documents(documents, None).await; assert_eq!(code, 202, "response: {}", response); - index.wait_task(1).await; + index.wait_task(response.uid()).await.succeeded(); - let (response, code) = index.get_task(1).await; + let (response, code) = index.get_task(response.uid()).await; assert_eq!(code, 200); assert_eq!(response["status"], "succeeded"); @@ -140,12 +139,12 @@ async fn update_document_gzip_encoded() { #[actix_rt::test] async fn update_larger_dataset() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(); - index.update_documents(documents, None).await; - index.wait_task(0).await; - let (response, code) = index.get_task(0).await; + let (task, _code) = index.update_documents(documents, None).await; + index.wait_task(task.uid()).await.succeeded(); + let (response, code) = index.get_task(task.uid()).await; assert_eq!(code, 200); assert_eq!(response["type"], "documentAdditionOrUpdate"); assert_eq!(response["details"]["indexedDocuments"], 77); @@ -158,8 +157,8 @@ async fn update_larger_dataset() { #[actix_rt::test] async fn error_update_documents_bad_document_id() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); index.create(Some("docid")).await; let documents = json!([ { @@ -167,13 +166,13 @@ async fn error_update_documents_bad_document_id() { "content": "foobar" } ]); - index.update_documents(documents, None).await; - let response = index.wait_task(1).await; + let (task, _code) = index.update_documents(documents, None).await; + let response = index.wait_task(task.uid()).await; assert_eq!(response["status"], json!("failed")); assert_eq!( response["error"]["message"], json!( - r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."# + r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_), and can not be more than 512 bytes."# ) ); assert_eq!(response["error"]["code"], json!("invalid_document_id")); @@ -186,8 +185,8 @@ async fn error_update_documents_bad_document_id() { #[actix_rt::test] async fn error_update_documents_missing_document_id() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); index.create(Some("docid")).await; let documents = json!([ { @@ -195,8 +194,8 @@ async fn error_update_documents_missing_document_id() { "content": "foobar" } ]); - index.update_documents(documents, None).await; - let response = index.wait_task(1).await; + let (task, _code) = index.update_documents(documents, None).await; + let response = index.wait_task(task.uid()).await; assert_eq!(response["status"], "failed"); assert_eq!( response["error"]["message"], @@ -212,8 +211,8 @@ async fn error_update_documents_missing_document_id() { #[actix_rt::test] async fn update_faceted_document() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (response, code) = index .update_settings(json!({ @@ -221,7 +220,7 @@ async fn update_faceted_document() { })) .await; assert_eq!("202", code.as_str(), "{:?}", response); - index.wait_task(0).await; + index.wait_task(response.uid()).await.succeeded(); let documents: Vec<_> = (0..1000) .map(|id| { @@ -232,10 +231,10 @@ async fn update_faceted_document() { }) .collect(); - let (_response, code) = index.add_documents(documents.into(), None).await; + let (response, code) = index.add_documents(documents.into(), None).await; assert_eq!(code, 202); - index.wait_task(1).await; + index.wait_task(response.uid()).await.succeeded(); let documents = json!([ { @@ -247,7 +246,7 @@ async fn update_faceted_document() { let (response, code) = index.update_documents(documents, None).await; assert_eq!(code, 202, "response: {}", response); - index.wait_task(2).await; + index.wait_task(response.uid()).await.succeeded(); index .search(json!({"limit": 10}), |response, code| { diff --git a/meilisearch/tests/dumps/data.rs b/crates/meilisearch/tests/dumps/data.rs similarity index 100% rename from meilisearch/tests/dumps/data.rs rename to crates/meilisearch/tests/dumps/data.rs diff --git a/meilisearch/tests/dumps/mod.rs b/crates/meilisearch/tests/dumps/mod.rs similarity index 100% rename from meilisearch/tests/dumps/mod.rs rename to crates/meilisearch/tests/dumps/mod.rs diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap similarity index 100% rename from meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap rename to crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap diff --git a/meilisearch/tests/features/mod.rs b/crates/meilisearch/tests/features/mod.rs similarity index 100% rename from meilisearch/tests/features/mod.rs rename to crates/meilisearch/tests/features/mod.rs diff --git a/meilisearch/tests/index/create_index.rs b/crates/meilisearch/tests/index/create_index.rs similarity index 74% rename from meilisearch/tests/index/create_index.rs rename to crates/meilisearch/tests/index/create_index.rs index b51ccab51..9b9fbd039 100644 --- a/meilisearch/tests/index/create_index.rs +++ b/crates/meilisearch/tests/index/create_index.rs @@ -9,15 +9,15 @@ use crate::json; #[actix_rt::test] async fn create_index_no_primary_key() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (response, code) = index.create(None).await; assert_eq!(code, 202); assert_eq!(response["status"], "enqueued"); - let response = index.wait_task(0).await; + let response = index.wait_task(response.uid()).await; assert_eq!(response["status"], "succeeded"); assert_eq!(response["type"], "indexCreation"); @@ -26,15 +26,15 @@ async fn create_index_no_primary_key() { #[actix_rt::test] async fn create_index_with_gzip_encoded_request() { - let server = Server::new().await; - let index = server.index_with_encoder("test", Encoder::Gzip); + let server = Server::new_shared(); + let index = server.unique_index_with_encoder(Encoder::Gzip); let (response, code) = index.create(None).await; assert_eq!(code, 202); assert_eq!(response["status"], "enqueued"); - let response = index.wait_task(0).await; + let response = index.wait_task(response.uid()).await; assert_eq!(response["status"], "succeeded"); assert_eq!(response["type"], "indexCreation"); @@ -43,7 +43,7 @@ async fn create_index_with_gzip_encoded_request() { #[actix_rt::test] async fn create_index_with_gzip_encoded_request_and_receiving_brotli_encoded_response() { - let server = Server::new().await; + let server = Server::new_shared(); let app = server.init_web_app().await; let body = serde_json::to_string(&json!({ @@ -68,21 +68,20 @@ async fn create_index_with_gzip_encoded_request_and_receiving_brotli_encoded_res let parsed_response = serde_json::from_slice::(decoded.into().as_ref()).expect("Expecting valid json"); - assert_eq!(parsed_response["taskUid"], 0); assert_eq!(parsed_response["indexUid"], "test"); } #[actix_rt::test] async fn create_index_with_zlib_encoded_request() { - let server = Server::new().await; - let index = server.index_with_encoder("test", Encoder::Deflate); + let server = Server::new_shared(); + let index = server.unique_index_with_encoder(Encoder::Deflate); let (response, code) = index.create(None).await; assert_eq!(code, 202); assert_eq!(response["status"], "enqueued"); - let response = index.wait_task(0).await; + let response = index.wait_task(response.uid()).await; assert_eq!(response["status"], "succeeded"); assert_eq!(response["type"], "indexCreation"); @@ -91,15 +90,15 @@ async fn create_index_with_zlib_encoded_request() { #[actix_rt::test] async fn create_index_with_brotli_encoded_request() { - let server = Server::new().await; - let index = server.index_with_encoder("test", Encoder::Brotli); + let server = Server::new_shared(); + let index = server.unique_index_with_encoder(Encoder::Brotli); let (response, code) = index.create(None).await; assert_eq!(code, 202); assert_eq!(response["status"], "enqueued"); - let response = index.wait_task(0).await; + let response = index.wait_task(response.uid()).await; assert_eq!(response["status"], "succeeded"); assert_eq!(response["type"], "indexCreation"); @@ -108,15 +107,15 @@ async fn create_index_with_brotli_encoded_request() { #[actix_rt::test] async fn create_index_with_primary_key() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (response, code) = index.create(Some("primary")).await; assert_eq!(code, 202); assert_eq!(response["status"], "enqueued"); - let response = index.wait_task(0).await; + let response = index.wait_task(response.uid()).await; assert_eq!(response["status"], "succeeded"); assert_eq!(response["type"], "indexCreation"); @@ -125,14 +124,25 @@ async fn create_index_with_primary_key() { #[actix_rt::test] async fn create_index_with_invalid_primary_key() { - let document = json!([ { "id": 2, "title": "Pride and Prejudice" } ]); + let documents = json!([ { "id": 2, "title": "Pride and Prejudice" } ]); - let server = Server::new().await; - let index = server.index("movies"); - let (_response, code) = index.add_documents(document, Some("title")).await; + let server = Server::new_shared(); + let index = server.unique_index(); + let (response, code) = index.add_documents(documents, Some("title")).await; assert_eq!(code, 202); - index.wait_task(0).await; + index.wait_task(response.uid()).await; + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], json!(null)); + + let documents = json!([ { "id": "e".repeat(513) } ]); + + let (response, code) = index.add_documents(documents, Some("id")).await; + assert_eq!(code, 202); + + index.wait_task(response.uid()).await; let (response, code) = index.get().await; assert_eq!(code, 200); @@ -141,19 +151,19 @@ async fn create_index_with_invalid_primary_key() { #[actix_rt::test] async fn test_create_multiple_indexes() { - let server = Server::new().await; - let index1 = server.index("test1"); - let index2 = server.index("test2"); - let index3 = server.index("test3"); - let index4 = server.index("test4"); + let server = Server::new_shared(); + let index1 = server.unique_index(); + let index2 = server.unique_index(); + let index3 = server.unique_index(); + let index4 = server.unique_index(); - index1.create(None).await; - index2.create(None).await; - index3.create(None).await; + let (task1, _) = index1.create(None).await; + let (task2, _) = index2.create(None).await; + let (task3, _) = index3.create(None).await; - index1.wait_task(0).await; - index1.wait_task(1).await; - index1.wait_task(2).await; + index1.wait_task(task1.uid()).await.succeeded(); + index2.wait_task(task2.uid()).await.succeeded(); + index3.wait_task(task3.uid()).await.succeeded(); assert_eq!(index1.get().await.1, 200); assert_eq!(index2.get().await.1, 200); @@ -163,18 +173,22 @@ async fn test_create_multiple_indexes() { #[actix_rt::test] async fn error_create_existing_index() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (_, code) = index.create(Some("primary")).await; assert_eq!(code, 202); - index.create(Some("primary")).await; + let (task, _) = index.create(Some("primary")).await; - let response = index.wait_task(1).await; + let response = index.wait_task(task.uid()).await; + let msg = format!( + "Index `{}` already exists.", + task["indexUid"].as_str().expect("indexUid should exist").trim_matches('"') + ); let expected_response = json!({ - "message": "Index `test` already exists.", + "message": msg, "code": "index_already_exists", "type": "invalid_request", "link":"https://docs.meilisearch.com/errors#index_already_exists" @@ -192,7 +206,7 @@ async fn error_create_with_invalid_index_uid() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "Invalid value at `.uid`: `test test#!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "Invalid value at `.uid`: `test test#!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" diff --git a/meilisearch/tests/index/delete_index.rs b/crates/meilisearch/tests/index/delete_index.rs similarity index 66% rename from meilisearch/tests/index/delete_index.rs rename to crates/meilisearch/tests/index/delete_index.rs index e404a6003..03185d21a 100644 --- a/meilisearch/tests/index/delete_index.rs +++ b/crates/meilisearch/tests/index/delete_index.rs @@ -1,51 +1,50 @@ -use crate::common::Server; +use crate::common::{shared_does_not_exists_index, Server}; use crate::json; #[actix_rt::test] async fn create_and_delete_index() { - let server = Server::new().await; - let index = server.index("test"); - let (_response, code) = index.create(None).await; + let server = Server::new_shared(); + let index = server.unique_index(); + let (response, code) = index.create(None).await; assert_eq!(code, 202); - index.wait_task(0).await; + index.wait_task(response.uid()).await.succeeded(); assert_eq!(index.get().await.1, 200); - let (_response, code) = index.delete().await; + let (response, code) = index.delete().await; assert_eq!(code, 202); - index.wait_task(1).await; + index.wait_task(response.uid()).await.succeeded(); assert_eq!(index.get().await.1, 404); } #[actix_rt::test] async fn error_delete_unexisting_index() { - let server = Server::new().await; - let index = server.index("test"); - let (_, code) = index.delete().await; + let index = shared_does_not_exists_index().await; + let (task, code) = index.delete_index_fail().await; assert_eq!(code, 202); let expected_response = json!({ - "message": "Index `test` not found.", + "message": "Index `DOES_NOT_EXISTS` not found.", "code": "index_not_found", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#index_not_found" }); - let response = index.wait_task(0).await; + let response = index.wait_task(task.uid()).await; assert_eq!(response["status"], "failed"); assert_eq!(response["error"], expected_response); } #[actix_rt::test] async fn loop_delete_add_documents() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let documents = json!([{"id": 1, "field1": "hello"}]); let mut tasks = Vec::new(); for _ in 0..50 { diff --git a/meilisearch/tests/index/errors.rs b/crates/meilisearch/tests/index/errors.rs similarity index 96% rename from meilisearch/tests/index/errors.rs rename to crates/meilisearch/tests/index/errors.rs index 9c677ee12..3bab83955 100644 --- a/meilisearch/tests/index/errors.rs +++ b/crates/meilisearch/tests/index/errors.rs @@ -75,7 +75,7 @@ async fn create_index_bad_uid() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "Invalid value at `.uid`: `the best doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "Invalid value at `.uid`: `the best doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -136,7 +136,7 @@ async fn get_index_bad_uid() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "`the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "`the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -232,7 +232,7 @@ async fn update_index_bad_uid() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "`the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "`the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -247,7 +247,7 @@ async fn delete_index_bad_uid() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "`the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "`the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" diff --git a/meilisearch/tests/index/get_index.rs b/crates/meilisearch/tests/index/get_index.rs similarity index 98% rename from meilisearch/tests/index/get_index.rs rename to crates/meilisearch/tests/index/get_index.rs index 5a184c8ce..ce08251be 100644 --- a/meilisearch/tests/index/get_index.rs +++ b/crates/meilisearch/tests/index/get_index.rs @@ -186,7 +186,7 @@ async fn get_invalid_index_uid() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "`this is not a valid index name` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "`this is not a valid index name` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" diff --git a/meilisearch/tests/index/mod.rs b/crates/meilisearch/tests/index/mod.rs similarity index 100% rename from meilisearch/tests/index/mod.rs rename to crates/meilisearch/tests/index/mod.rs diff --git a/meilisearch/tests/index/stats.rs b/crates/meilisearch/tests/index/stats.rs similarity index 100% rename from meilisearch/tests/index/stats.rs rename to crates/meilisearch/tests/index/stats.rs diff --git a/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs similarity index 100% rename from meilisearch/tests/index/update_index.rs rename to crates/meilisearch/tests/index/update_index.rs diff --git a/meilisearch/tests/integration.rs b/crates/meilisearch/tests/integration.rs similarity index 100% rename from meilisearch/tests/integration.rs rename to crates/meilisearch/tests/integration.rs diff --git a/meilisearch/tests/logs/error.rs b/crates/meilisearch/tests/logs/error.rs similarity index 100% rename from meilisearch/tests/logs/error.rs rename to crates/meilisearch/tests/logs/error.rs diff --git a/meilisearch/tests/logs/mod.rs b/crates/meilisearch/tests/logs/mod.rs similarity index 96% rename from meilisearch/tests/logs/mod.rs rename to crates/meilisearch/tests/logs/mod.rs index 9f4649dca..26482b561 100644 --- a/meilisearch/tests/logs/mod.rs +++ b/crates/meilisearch/tests/logs/mod.rs @@ -7,8 +7,9 @@ use std::str::FromStr; use actix_web::http::header::ContentType; use actix_web::web::Data; use meili_snap::snapshot; +use meilisearch::analytics::Analytics; use meilisearch::search_queue::SearchQueue; -use meilisearch::{analytics, create_app, Opt, SubscriberForSecondLayer}; +use meilisearch::{create_app, Opt, SubscriberForSecondLayer}; use tracing::level_filters::LevelFilter; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::Layer; @@ -54,7 +55,7 @@ async fn basic_test_log_stream_route() { Data::new(search_queue), server.service.options.clone(), (route_layer_handle, stderr_layer_handle), - analytics::MockAnalytics::new(&server.service.options), + Data::new(Analytics::no_analytics()), true, )) .await; diff --git a/meilisearch/tests/search/distinct.rs b/crates/meilisearch/tests/search/distinct.rs similarity index 100% rename from meilisearch/tests/search/distinct.rs rename to crates/meilisearch/tests/search/distinct.rs diff --git a/meilisearch/tests/search/errors.rs b/crates/meilisearch/tests/search/errors.rs similarity index 100% rename from meilisearch/tests/search/errors.rs rename to crates/meilisearch/tests/search/errors.rs diff --git a/meilisearch/tests/search/facet_search.rs b/crates/meilisearch/tests/search/facet_search.rs similarity index 100% rename from meilisearch/tests/search/facet_search.rs rename to crates/meilisearch/tests/search/facet_search.rs diff --git a/meilisearch/tests/search/formatted.rs b/crates/meilisearch/tests/search/formatted.rs similarity index 100% rename from meilisearch/tests/search/formatted.rs rename to crates/meilisearch/tests/search/formatted.rs diff --git a/meilisearch/tests/search/geo.rs b/crates/meilisearch/tests/search/geo.rs similarity index 100% rename from meilisearch/tests/search/geo.rs rename to crates/meilisearch/tests/search/geo.rs diff --git a/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs similarity index 95% rename from meilisearch/tests/search/hybrid.rs rename to crates/meilisearch/tests/search/hybrid.rs index e301c0b05..00a65d9aa 100644 --- a/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -568,6 +568,57 @@ async fn retrieve_vectors() { ] "###); + // use explicit `_vectors` in displayed attributes + let (response, code) = index + .update_settings(json!({ "displayedAttributes": ["id", "title", "desc", "_vectors"]} )) + .await; + assert_eq!(202, code, "{:?}", response); + index.wait_task(response.uid()).await; + + let (response, code) = index + .search_post( + json!({"q": "Captain", "hybrid": {"embedder": "default", "semanticRatio": 0.2}, "retrieveVectors": true}), + ) + .await; + snapshot!(code, @"200 OK"); + insta::assert_json_snapshot!(response["hits"], {"[]._vectors.default.embeddings" => "[vectors]"}, @r###" + [ + { + "title": "Captain Planet", + "desc": "He's not part of the Marvel Cinematic Universe", + "id": "2", + "_vectors": { + "default": { + "embeddings": "[vectors]", + "regenerate": true + } + } + }, + { + "title": "Captain Marvel", + "desc": "a Shazam ersatz", + "id": "3", + "_vectors": { + "default": { + "embeddings": "[vectors]", + "regenerate": true + } + } + }, + { + "title": "Shazam!", + "desc": "a Captain Marvel ersatz", + "id": "1", + "_vectors": { + "default": { + "embeddings": "[vectors]", + "regenerate": true + } + } + } + ] + "###); + // remove `_vectors` from displayed attributes let (response, code) = index.update_settings(json!({ "displayedAttributes": ["id", "title", "desc"]} )).await; diff --git a/meilisearch/tests/search/locales.rs b/crates/meilisearch/tests/search/locales.rs similarity index 100% rename from meilisearch/tests/search/locales.rs rename to crates/meilisearch/tests/search/locales.rs diff --git a/meilisearch/tests/search/matching_strategy.rs b/crates/meilisearch/tests/search/matching_strategy.rs similarity index 100% rename from meilisearch/tests/search/matching_strategy.rs rename to crates/meilisearch/tests/search/matching_strategy.rs diff --git a/meilisearch/tests/search/mod.rs b/crates/meilisearch/tests/search/mod.rs similarity index 100% rename from meilisearch/tests/search/mod.rs rename to crates/meilisearch/tests/search/mod.rs diff --git a/meilisearch/tests/search/multi.rs b/crates/meilisearch/tests/search/multi.rs similarity index 99% rename from meilisearch/tests/search/multi.rs rename to crates/meilisearch/tests/search/multi.rs index b9593f05f..eaa1da15f 100644 --- a/meilisearch/tests/search/multi.rs +++ b/crates/meilisearch/tests/search/multi.rs @@ -412,7 +412,7 @@ async fn simple_search_illegal_index_uid() { snapshot!(code, @"400 Bad Request"); insta::assert_json_snapshot!(response, @r###" { - "message": "Invalid value at `.queries[0].indexUid`: `hé` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "Invalid value at `.queries[0].indexUid`: `hé` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -437,7 +437,7 @@ async fn federation_search_illegal_index_uid() { snapshot!(code, @"400 Bad Request"); insta::assert_json_snapshot!(response, @r###" { - "message": "Invalid value at `.queries[0].indexUid`: `hé` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "Invalid value at `.queries[0].indexUid`: `hé` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" diff --git a/meilisearch/tests/search/pagination.rs b/crates/meilisearch/tests/search/pagination.rs similarity index 100% rename from meilisearch/tests/search/pagination.rs rename to crates/meilisearch/tests/search/pagination.rs diff --git a/meilisearch/tests/search/restrict_searchable.rs b/crates/meilisearch/tests/search/restrict_searchable.rs similarity index 100% rename from meilisearch/tests/search/restrict_searchable.rs rename to crates/meilisearch/tests/search/restrict_searchable.rs diff --git a/meilisearch/tests/search/search_queue.rs b/crates/meilisearch/tests/search/search_queue.rs similarity index 100% rename from meilisearch/tests/search/search_queue.rs rename to crates/meilisearch/tests/search/search_queue.rs diff --git a/meilisearch/tests/search/snapshots/distinct.rs/distinct_at_search_time/succeed.snap b/crates/meilisearch/tests/search/snapshots/distinct.rs/distinct_at_search_time/succeed.snap similarity index 100% rename from meilisearch/tests/search/snapshots/distinct.rs/distinct_at_search_time/succeed.snap rename to crates/meilisearch/tests/search/snapshots/distinct.rs/distinct_at_search_time/succeed.snap diff --git a/meilisearch/tests/search/snapshots/errors.rs/distinct_at_search_time/task-succeed.snap b/crates/meilisearch/tests/search/snapshots/errors.rs/distinct_at_search_time/task-succeed.snap similarity index 100% rename from meilisearch/tests/search/snapshots/errors.rs/distinct_at_search_time/task-succeed.snap rename to crates/meilisearch/tests/search/snapshots/errors.rs/distinct_at_search_time/task-succeed.snap diff --git a/meilisearch/tests/settings/distinct.rs b/crates/meilisearch/tests/settings/distinct.rs similarity index 100% rename from meilisearch/tests/settings/distinct.rs rename to crates/meilisearch/tests/settings/distinct.rs diff --git a/meilisearch/tests/settings/errors.rs b/crates/meilisearch/tests/settings/errors.rs similarity index 100% rename from meilisearch/tests/settings/errors.rs rename to crates/meilisearch/tests/settings/errors.rs diff --git a/meilisearch/tests/settings/get_settings.rs b/crates/meilisearch/tests/settings/get_settings.rs similarity index 99% rename from meilisearch/tests/settings/get_settings.rs rename to crates/meilisearch/tests/settings/get_settings.rs index e99a9fa65..6de0db0b3 100644 --- a/meilisearch/tests/settings/get_settings.rs +++ b/crates/meilisearch/tests/settings/get_settings.rs @@ -330,7 +330,7 @@ async fn error_update_setting_unexisting_index_invalid_uid() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { - "message": "`test##! ` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "`test##! ` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" diff --git a/meilisearch/tests/settings/mod.rs b/crates/meilisearch/tests/settings/mod.rs similarity index 100% rename from meilisearch/tests/settings/mod.rs rename to crates/meilisearch/tests/settings/mod.rs diff --git a/meilisearch/tests/settings/proximity_settings.rs b/crates/meilisearch/tests/settings/proximity_settings.rs similarity index 100% rename from meilisearch/tests/settings/proximity_settings.rs rename to crates/meilisearch/tests/settings/proximity_settings.rs diff --git a/meilisearch/tests/settings/tokenizer_customization.rs b/crates/meilisearch/tests/settings/tokenizer_customization.rs similarity index 100% rename from meilisearch/tests/settings/tokenizer_customization.rs rename to crates/meilisearch/tests/settings/tokenizer_customization.rs diff --git a/meilisearch/tests/similar/errors.rs b/crates/meilisearch/tests/similar/errors.rs similarity index 99% rename from meilisearch/tests/similar/errors.rs rename to crates/meilisearch/tests/similar/errors.rs index 228358d54..1e933e1c0 100644 --- a/meilisearch/tests/similar/errors.rs +++ b/crates/meilisearch/tests/similar/errors.rs @@ -79,7 +79,7 @@ async fn similar_bad_id() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "Invalid value at `.id`: the value of `id` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_).", + "message": "Invalid value at `.id`: the value of `id` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_similar_id", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_similar_id" @@ -172,7 +172,7 @@ async fn similar_invalid_id() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "Invalid value at `.id`: the value of `id` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_).", + "message": "Invalid value at `.id`: the value of `id` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_similar_id", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_similar_id" diff --git a/meilisearch/tests/similar/mod.rs b/crates/meilisearch/tests/similar/mod.rs similarity index 100% rename from meilisearch/tests/similar/mod.rs rename to crates/meilisearch/tests/similar/mod.rs diff --git a/meilisearch/tests/snapshot/mod.rs b/crates/meilisearch/tests/snapshot/mod.rs similarity index 100% rename from meilisearch/tests/snapshot/mod.rs rename to crates/meilisearch/tests/snapshot/mod.rs diff --git a/meilisearch/tests/stats/mod.rs b/crates/meilisearch/tests/stats/mod.rs similarity index 100% rename from meilisearch/tests/stats/mod.rs rename to crates/meilisearch/tests/stats/mod.rs diff --git a/meilisearch/tests/swap_indexes/errors.rs b/crates/meilisearch/tests/swap_indexes/errors.rs similarity index 100% rename from meilisearch/tests/swap_indexes/errors.rs rename to crates/meilisearch/tests/swap_indexes/errors.rs diff --git a/meilisearch/tests/swap_indexes/mod.rs b/crates/meilisearch/tests/swap_indexes/mod.rs similarity index 100% rename from meilisearch/tests/swap_indexes/mod.rs rename to crates/meilisearch/tests/swap_indexes/mod.rs diff --git a/meilisearch/tests/tasks/errors.rs b/crates/meilisearch/tests/tasks/errors.rs similarity index 99% rename from meilisearch/tests/tasks/errors.rs rename to crates/meilisearch/tests/tasks/errors.rs index c404a2329..42ec42997 100644 --- a/meilisearch/tests/tasks/errors.rs +++ b/crates/meilisearch/tests/tasks/errors.rs @@ -173,7 +173,7 @@ async fn task_bad_index_uids() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "Invalid value in parameter `indexUids`: `the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "Invalid value in parameter `indexUids`: `the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -184,7 +184,7 @@ async fn task_bad_index_uids() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "Invalid value in parameter `indexUids`: `the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "Invalid value in parameter `indexUids`: `the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" @@ -195,7 +195,7 @@ async fn task_bad_index_uids() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "Invalid value in parameter `indexUids`: `the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).", + "message": "Invalid value in parameter `indexUids`: `the good doggo` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes.", "code": "invalid_index_uid", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_index_uid" diff --git a/meilisearch/tests/tasks/mod.rs b/crates/meilisearch/tests/tasks/mod.rs similarity index 100% rename from meilisearch/tests/tasks/mod.rs rename to crates/meilisearch/tests/tasks/mod.rs diff --git a/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs similarity index 100% rename from meilisearch/tests/tasks/webhook.rs rename to crates/meilisearch/tests/tasks/webhook.rs diff --git a/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs similarity index 100% rename from meilisearch/tests/vector/binary_quantized.rs rename to crates/meilisearch/tests/vector/binary_quantized.rs diff --git a/meilisearch/tests/vector/intel_gen.txt.gz b/crates/meilisearch/tests/vector/intel_gen.txt.gz similarity index 100% rename from meilisearch/tests/vector/intel_gen.txt.gz rename to crates/meilisearch/tests/vector/intel_gen.txt.gz diff --git a/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs similarity index 100% rename from meilisearch/tests/vector/mod.rs rename to crates/meilisearch/tests/vector/mod.rs diff --git a/meilisearch/tests/vector/openai.rs b/crates/meilisearch/tests/vector/openai.rs similarity index 100% rename from meilisearch/tests/vector/openai.rs rename to crates/meilisearch/tests/vector/openai.rs diff --git a/meilisearch/tests/vector/openai_responses.json.gz b/crates/meilisearch/tests/vector/openai_responses.json.gz similarity index 100% rename from meilisearch/tests/vector/openai_responses.json.gz rename to crates/meilisearch/tests/vector/openai_responses.json.gz diff --git a/meilisearch/tests/vector/openai_tokenized_responses.json.gz b/crates/meilisearch/tests/vector/openai_tokenized_responses.json.gz similarity index 100% rename from meilisearch/tests/vector/openai_tokenized_responses.json.gz rename to crates/meilisearch/tests/vector/openai_tokenized_responses.json.gz diff --git a/meilisearch/tests/vector/rest.rs b/crates/meilisearch/tests/vector/rest.rs similarity index 100% rename from meilisearch/tests/vector/rest.rs rename to crates/meilisearch/tests/vector/rest.rs diff --git a/meilisearch/tests/vector/settings.rs b/crates/meilisearch/tests/vector/settings.rs similarity index 74% rename from meilisearch/tests/vector/settings.rs rename to crates/meilisearch/tests/vector/settings.rs index 4f07ca18b..ed45913a8 100644 --- a/meilisearch/tests/vector/settings.rs +++ b/crates/meilisearch/tests/vector/settings.rs @@ -4,6 +4,53 @@ use crate::common::{GetAllDocumentsOptions, Server}; use crate::json; use crate::vector::generate_default_user_provided_documents; +#[actix_rt::test] +async fn field_unavailable_for_source() { + let server = Server::new().await; + let index = server.index("doggo"); + let (value, code) = server.set_features(json!({"vectorStore": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r###" + { + "vectorStore": true, + "metrics": false, + "logsRoute": false, + "editDocumentsByFunction": false, + "containsFilter": false + } + "###); + + let (response, code) = index + .update_settings(json!({ + "embedders": { "manual": {"source": "userProvided", "documentTemplate": "{{doc.documentTemplate}}"}}, + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(response, @r###" + { + "message": "`.embedders.manual`: Field `documentTemplate` unavailable for source `userProvided` (only available for sources: `huggingFace`, `openAi`, `ollama`, `rest`). Available fields: `source`, `dimensions`, `distribution`, `binaryQuantized`", + "code": "invalid_settings_embedders", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_settings_embedders" + } + "###); + + let (response, code) = index + .update_settings(json!({ + "embedders": { "default": {"source": "openAi", "revision": "42"}}, + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(response, @r###" + { + "message": "`.embedders.default`: Field `revision` unavailable for source `openAi` (only available for sources: `huggingFace`). Available fields: `source`, `model`, `apiKey`, `documentTemplate`, `dimensions`, `distribution`, `url`, `binaryQuantized`", + "code": "invalid_settings_embedders", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_settings_embedders" + } + "###); +} + #[actix_rt::test] async fn update_embedder() { let server = Server::new().await; diff --git a/meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/document-added.snap b/crates/meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/document-added.snap similarity index 100% rename from meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/document-added.snap rename to crates/meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/document-added.snap diff --git a/meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/document-deleted.snap b/crates/meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/document-deleted.snap similarity index 100% rename from meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/document-deleted.snap rename to crates/meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/document-deleted.snap diff --git a/meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/settings-processed.snap b/crates/meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/settings-processed.snap similarity index 100% rename from meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/settings-processed.snap rename to crates/meilisearch/tests/vector/snapshots/mod.rs/add_remove_one_vector_4588/settings-processed.snap diff --git a/meilitool/Cargo.toml b/crates/meilitool/Cargo.toml similarity index 75% rename from meilitool/Cargo.toml rename to crates/meilitool/Cargo.toml index ce6c1ad5b..048da6232 100644 --- a/meilitool/Cargo.toml +++ b/crates/meilitool/Cargo.toml @@ -16,5 +16,6 @@ file-store = { path = "../file-store" } meilisearch-auth = { path = "../meilisearch-auth" } meilisearch-types = { path = "../meilisearch-types" } serde = { version = "1.0.209", features = ["derive"] } -time = { version = "0.3.36", features = ["formatting"] } +time = { version = "0.3.36", features = ["formatting", "parsing", "alloc"] } uuid = { version = "1.10.0", features = ["v4"], default-features = false } +arroy_v04_to_v05 = { package = "arroy", git = "https://github.com/meilisearch/arroy/", tag = "DO-NOT-DELETE-upgrade-v04-to-v05" } diff --git a/crates/meilitool/src/main.rs b/crates/meilitool/src/main.rs new file mode 100644 index 000000000..978824356 --- /dev/null +++ b/crates/meilitool/src/main.rs @@ -0,0 +1,334 @@ +use std::fs::{read_dir, read_to_string, remove_file, File}; +use std::io::BufWriter; +use std::path::PathBuf; + +use anyhow::Context; +use clap::{Parser, Subcommand}; +use dump::{DumpWriter, IndexMetadata}; +use file_store::FileStore; +use meilisearch_auth::AuthController; +use meilisearch_types::heed::types::{SerdeJson, Str}; +use meilisearch_types::heed::{Database, Env, EnvOpenOptions, RoTxn, RwTxn, Unspecified}; +use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader}; +use meilisearch_types::milli::{obkv_to_json, BEU32}; +use meilisearch_types::tasks::{Status, Task}; +use meilisearch_types::versioning::{get_version, parse_version}; +use meilisearch_types::Index; +use time::macros::format_description; +use time::OffsetDateTime; +use upgrade::OfflineUpgrade; +use uuid_codec::UuidCodec; + +mod upgrade; +mod uuid_codec; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Cli { + /// The database path where the Meilisearch is running. + #[arg(long, default_value = "data.ms/")] + db_path: PathBuf, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Clears the task queue and make it empty. + /// + /// This command can be safely executed even if Meilisearch is running and processing tasks. + /// Once the task queue is empty you can restart Meilisearch and no more tasks must be visible, + /// even the ones that were processing. However, it's highly possible that you see the processing + /// tasks in the queue again with an associated internal error message. + ClearTaskQueue, + + /// Exports a dump from the Meilisearch database. + /// + /// Make sure to run this command when Meilisearch is not running or running but not processing tasks. + /// If tasks are being processed while a dump is being exported there are chances for the dump to be + /// malformed with missing tasks. + /// + /// TODO Verify this claim or make sure it cannot happen and we can export dumps + /// without caring about killing Meilisearch first! + ExportADump { + /// The directory in which the dump will be created. + #[arg(long, default_value = "dumps/")] + dump_dir: PathBuf, + + /// Skip dumping the enqueued or processing tasks. + /// + /// Can be useful when there are a lot of them and it is not particularly useful + /// to keep them. Note that only the enqueued tasks takes up space so skipping + /// the processed ones is not particularly interesting. + #[arg(long)] + skip_enqueued_tasks: bool, + }, + + /// Attempts to upgrade from one major version to the next without a dump. + /// + /// Make sure to run this commmand when Meilisearch is not running! + /// If Meilisearch is running while executing this command, the database could be corrupted + /// (contain data from both the old and the new versions) + /// + /// Supported upgrade paths: + /// + /// - v1.9.x -> v1.10.x -> v1.11.x + OfflineUpgrade { + #[arg(long)] + target_version: String, + }, +} + +fn main() -> anyhow::Result<()> { + let Cli { db_path, command } = Cli::parse(); + + let detected_version = get_version(&db_path).context("While checking the version file")?; + + match command { + Command::ClearTaskQueue => clear_task_queue(db_path), + Command::ExportADump { dump_dir, skip_enqueued_tasks } => { + export_a_dump(db_path, dump_dir, skip_enqueued_tasks) + } + Command::OfflineUpgrade { target_version } => { + let target_version = parse_version(&target_version).context("While parsing `--target-version`. Make sure `--target-version` is in the format MAJOR.MINOR.PATCH")?; + OfflineUpgrade { db_path, current_version: detected_version, target_version }.upgrade() + } + } +} + +/// Clears the task queue located at `db_path`. +fn clear_task_queue(db_path: PathBuf) -> anyhow::Result<()> { + let path = db_path.join("tasks"); + let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&path) } + .with_context(|| format!("While trying to open {:?}", path.display()))?; + + eprintln!("Deleting tasks from the database..."); + + let mut wtxn = env.write_txn()?; + let all_tasks = try_opening_poly_database(&env, &wtxn, "all-tasks")?; + let total = all_tasks.len(&wtxn)?; + let status = try_opening_poly_database(&env, &wtxn, "status")?; + let kind = try_opening_poly_database(&env, &wtxn, "kind")?; + let index_tasks = try_opening_poly_database(&env, &wtxn, "index-tasks")?; + let canceled_by = try_opening_poly_database(&env, &wtxn, "canceled_by")?; + let enqueued_at = try_opening_poly_database(&env, &wtxn, "enqueued-at")?; + let started_at = try_opening_poly_database(&env, &wtxn, "started-at")?; + let finished_at = try_opening_poly_database(&env, &wtxn, "finished-at")?; + + try_clearing_poly_database(&mut wtxn, all_tasks, "all-tasks")?; + try_clearing_poly_database(&mut wtxn, status, "status")?; + try_clearing_poly_database(&mut wtxn, kind, "kind")?; + try_clearing_poly_database(&mut wtxn, index_tasks, "index-tasks")?; + try_clearing_poly_database(&mut wtxn, canceled_by, "canceled_by")?; + try_clearing_poly_database(&mut wtxn, enqueued_at, "enqueued-at")?; + try_clearing_poly_database(&mut wtxn, started_at, "started-at")?; + try_clearing_poly_database(&mut wtxn, finished_at, "finished-at")?; + + wtxn.commit().context("While committing the transaction")?; + + eprintln!("Successfully deleted {total} tasks from the tasks database!"); + eprintln!("Deleting the content files from disk..."); + + let mut count = 0usize; + let update_files = db_path.join("update_files"); + let entries = read_dir(&update_files).with_context(|| { + format!("While trying to read the content of {:?}", update_files.display()) + })?; + for result in entries { + match result { + Ok(ent) => match remove_file(ent.path()) { + Ok(_) => count += 1, + Err(e) => eprintln!("Error while deleting {:?}: {}", ent.path().display(), e), + }, + Err(e) => { + eprintln!("Error while reading a file in {:?}: {}", update_files.display(), e) + } + } + } + + eprintln!("Successfully deleted {count} content files from disk!"); + + Ok(()) +} + +fn try_opening_database( + env: &Env, + rtxn: &RoTxn, + db_name: &str, +) -> anyhow::Result> { + env.open_database(rtxn, Some(db_name)) + .with_context(|| format!("While opening the {db_name:?} database"))? + .with_context(|| format!("Missing the {db_name:?} database")) +} + +fn try_opening_poly_database( + env: &Env, + rtxn: &RoTxn, + db_name: &str, +) -> anyhow::Result> { + env.database_options() + .name(db_name) + .open(rtxn) + .with_context(|| format!("While opening the {db_name:?} poly database"))? + .with_context(|| format!("Missing the {db_name:?} poly database")) +} + +fn try_clearing_poly_database( + wtxn: &mut RwTxn, + database: Database, + db_name: &str, +) -> anyhow::Result<()> { + database.clear(wtxn).with_context(|| format!("While clearing the {db_name:?} database")) +} + +/// Exports a dump into the dump directory. +fn export_a_dump( + db_path: PathBuf, + dump_dir: PathBuf, + skip_enqueued_tasks: bool, +) -> Result<(), anyhow::Error> { + let started_at = OffsetDateTime::now_utc(); + + // 1. Extracts the instance UID from disk + let instance_uid_path = db_path.join("instance-uid"); + let instance_uid = match read_to_string(&instance_uid_path) { + Ok(content) => match content.trim().parse() { + Ok(uuid) => Some(uuid), + Err(e) => { + eprintln!("Impossible to parse instance-uid: {e}"); + None + } + }, + Err(e) => { + eprintln!("Impossible to read {}: {}", instance_uid_path.display(), e); + None + } + }; + + let dump = DumpWriter::new(instance_uid).context("While creating a new dump")?; + let file_store = + FileStore::new(db_path.join("update_files")).context("While opening the FileStore")?; + + let index_scheduler_path = db_path.join("tasks"); + let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&index_scheduler_path) } + .with_context(|| format!("While trying to open {:?}", index_scheduler_path.display()))?; + + eprintln!("Dumping the keys..."); + + // 2. dump the keys + let auth_store = AuthController::new(&db_path, &None) + .with_context(|| format!("While opening the auth store at {}", db_path.display()))?; + let mut dump_keys = dump.create_keys()?; + let mut count = 0; + for key in auth_store.list_keys()? { + dump_keys.push_key(&key)?; + count += 1; + } + dump_keys.flush()?; + + eprintln!("Successfully dumped {count} keys!"); + + let rtxn = env.read_txn()?; + let all_tasks: Database> = + try_opening_database(&env, &rtxn, "all-tasks")?; + let index_mapping: Database = + try_opening_database(&env, &rtxn, "index-mapping")?; + + if skip_enqueued_tasks { + eprintln!("Skip dumping the enqueued tasks..."); + } else { + eprintln!("Dumping the enqueued tasks..."); + + // 3. dump the tasks + let mut dump_tasks = dump.create_tasks_queue()?; + let mut count = 0; + for ret in all_tasks.iter(&rtxn)? { + let (_, t) = ret?; + let status = t.status; + let content_file = t.content_uuid(); + let mut dump_content_file = dump_tasks.push_task(&t.into())?; + + // 3.1. Dump the `content_file` associated with the task if there is one and the task is not finished yet. + if let Some(content_file_uuid) = content_file { + if status == Status::Enqueued { + let content_file = file_store.get_update(content_file_uuid)?; + + let reader = + DocumentsBatchReader::from_reader(content_file).with_context(|| { + format!("While reading content file {:?}", content_file_uuid) + })?; + + let (mut cursor, documents_batch_index) = reader.into_cursor_and_fields_index(); + while let Some(doc) = cursor.next_document().with_context(|| { + format!("While iterating on content file {:?}", content_file_uuid) + })? { + dump_content_file + .push_document(&obkv_to_object(&doc, &documents_batch_index)?)?; + } + dump_content_file.flush()?; + count += 1; + } + } + } + dump_tasks.flush()?; + + eprintln!("Successfully dumped {count} enqueued tasks!"); + } + + eprintln!("Dumping the indexes..."); + + // 4. Dump the indexes + let mut count = 0; + for result in index_mapping.iter(&rtxn)? { + let (uid, uuid) = result?; + let index_path = db_path.join("indexes").join(uuid.to_string()); + let index = Index::new(EnvOpenOptions::new(), &index_path).with_context(|| { + format!("While trying to open the index at path {:?}", index_path.display()) + })?; + + let rtxn = index.read_txn()?; + let metadata = IndexMetadata { + uid: uid.to_owned(), + primary_key: index.primary_key(&rtxn)?.map(String::from), + created_at: index.created_at(&rtxn)?, + updated_at: index.updated_at(&rtxn)?, + }; + let mut index_dumper = dump.create_index(uid, &metadata)?; + + let fields_ids_map = index.fields_ids_map(&rtxn)?; + let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); + + // 4.1. Dump the documents + for ret in index.all_documents(&rtxn)? { + let (_id, doc) = ret?; + let document = obkv_to_json(&all_fields, &fields_ids_map, doc)?; + index_dumper.push_document(&document)?; + } + + // 4.2. Dump the settings + let settings = meilisearch_types::settings::settings( + &index, + &rtxn, + meilisearch_types::settings::SecretPolicy::RevealSecrets, + )?; + index_dumper.settings(&settings)?; + count += 1; + } + + eprintln!("Successfully dumped {count} indexes!"); + // We will not dump experimental feature settings + eprintln!("The tool is not dumping experimental features, please set them by hand afterward"); + + let dump_uid = started_at.format(format_description!( + "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" + )).unwrap(); + + let path = dump_dir.join(format!("{}.dump", dump_uid)); + let file = File::create(&path)?; + dump.persist_to(BufWriter::new(file))?; + + eprintln!("Dump exported at path {:?}", path.display()); + + Ok(()) +} diff --git a/crates/meilitool/src/upgrade/mod.rs b/crates/meilitool/src/upgrade/mod.rs new file mode 100644 index 000000000..36630c3b3 --- /dev/null +++ b/crates/meilitool/src/upgrade/mod.rs @@ -0,0 +1,73 @@ +mod v1_10; +mod v1_11; +mod v1_9; + +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context}; +use meilisearch_types::versioning::create_version_file; + +use v1_10::v1_9_to_v1_10; + +use crate::upgrade::v1_11::v1_10_to_v1_11; + +pub struct OfflineUpgrade { + pub db_path: PathBuf, + pub current_version: (String, String, String), + pub target_version: (String, String, String), +} + +impl OfflineUpgrade { + pub fn upgrade(self) -> anyhow::Result<()> { + let upgrade_list = [ + (v1_9_to_v1_10 as fn(&Path) -> Result<(), anyhow::Error>, "1", "10", "0"), + (v1_10_to_v1_11, "1", "11", "0"), + ]; + + let (current_major, current_minor, current_patch) = &self.current_version; + + let start_at = match ( + current_major.as_str(), + current_minor.as_str(), + current_patch.as_str(), + ) { + ("1", "9", _) => 0, + ("1", "10", _) => 1, + _ => { + bail!("Unsupported current version {current_major}.{current_minor}.{current_patch}. Can only upgrade from v1.9 and v1.10") + } + }; + + let (target_major, target_minor, target_patch) = &self.target_version; + + let ends_at = match (target_major.as_str(), target_minor.as_str(), target_patch.as_str()) { + ("1", "10", _) => 0, + ("1", "11", _) => 1, + (major, _, _) if major.starts_with('v') => { + bail!("Target version must not starts with a `v`. Instead of writing `v1.9.0` write `1.9.0` for example.") + } + _ => { + bail!("Unsupported target version {target_major}.{target_minor}.{target_patch}. Can only upgrade to v1.10 and v1.11") + } + }; + + println!("Starting the upgrade from {current_major}.{current_minor}.{current_patch} to {target_major}.{target_minor}.{target_patch}"); + + #[allow(clippy::needless_range_loop)] + for index in start_at..=ends_at { + let (func, major, minor, patch) = upgrade_list[index]; + (func)(&self.db_path)?; + println!("Done"); + // We're writing the version file just in case an issue arise _while_ upgrading. + // We don't want the DB to fail in an unknown state. + println!("Writing VERSION file"); + + create_version_file(&self.db_path, major, minor, patch) + .context("while writing VERSION file after the upgrade")?; + } + + println!("Success"); + + Ok(()) + } +} diff --git a/crates/meilitool/src/upgrade/v1_10.rs b/crates/meilitool/src/upgrade/v1_10.rs new file mode 100644 index 000000000..3dd7c72a2 --- /dev/null +++ b/crates/meilitool/src/upgrade/v1_10.rs @@ -0,0 +1,289 @@ +use anyhow::bail; +use std::path::Path; + +use anyhow::Context; +use meilisearch_types::{ + heed::{ + types::{SerdeJson, Str}, + Database, Env, EnvOpenOptions, RoTxn, RwTxn, Unspecified, + }, + milli::index::{db_name, main_key}, +}; + +use crate::{try_opening_database, try_opening_poly_database, uuid_codec::UuidCodec}; + +use super::v1_9; + +pub type FieldDistribution = std::collections::BTreeMap; + +/// The statistics that can be computed from an `Index` object. +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct IndexStats { + /// Number of documents in the index. + pub number_of_documents: u64, + /// Size taken up by the index' DB, in bytes. + /// + /// This includes the size taken by both the used and free pages of the DB, and as the free pages + /// are not returned to the disk after a deletion, this number is typically larger than + /// `used_database_size` that only includes the size of the used pages. + pub database_size: u64, + /// Size taken by the used pages of the index' DB, in bytes. + /// + /// As the DB backend does not return to the disk the pages that are not currently used by the DB, + /// this value is typically smaller than `database_size`. + pub used_database_size: u64, + /// Association of every field name with the number of times it occurs in the documents. + pub field_distribution: FieldDistribution, + /// Creation date of the index. + #[serde(with = "time::serde::rfc3339")] + pub created_at: time::OffsetDateTime, + /// Date of the last update of the index. + #[serde(with = "time::serde::rfc3339")] + pub updated_at: time::OffsetDateTime, +} + +impl From for IndexStats { + fn from( + v1_9::IndexStats { + number_of_documents, + database_size, + used_database_size, + field_distribution, + created_at, + updated_at, + }: v1_9::IndexStats, + ) -> Self { + IndexStats { + number_of_documents, + database_size, + used_database_size, + field_distribution, + created_at: created_at.0, + updated_at: updated_at.0, + } + } +} + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(transparent)] +pub struct OffsetDateTime(#[serde(with = "time::serde::rfc3339")] pub time::OffsetDateTime); + +fn update_index_stats( + index_stats: Database, + index_uid: &str, + index_uuid: uuid::Uuid, + sched_wtxn: &mut RwTxn, +) -> anyhow::Result<()> { + let ctx = || format!("while updating index stats for index `{index_uid}`"); + + let stats: Option<&str> = index_stats + .remap_data_type::() + .get(sched_wtxn, &index_uuid) + .with_context(ctx) + .with_context(|| "While reading value")?; + dbg!(stats); + + let stats: Option = index_stats + .remap_data_type::>() + .get(sched_wtxn, &index_uuid) + .with_context(ctx) + .with_context(|| "While reading value")?; + + if let Some(stats) = stats { + let stats: self::IndexStats = stats.into(); + + index_stats + .remap_data_type::>() + .put(sched_wtxn, &index_uuid, &stats) + .with_context(ctx) + .with_context(|| "While writing value")?; + } + + Ok(()) +} + +fn update_date_format( + index_uid: &str, + index_env: &Env, + index_wtxn: &mut RwTxn, +) -> anyhow::Result<()> { + let main = try_opening_poly_database(index_env, index_wtxn, db_name::MAIN) + .with_context(|| format!("while updating date format for index `{index_uid}`"))?; + + date_round_trip(index_wtxn, index_uid, main, main_key::CREATED_AT_KEY)?; + date_round_trip(index_wtxn, index_uid, main, main_key::UPDATED_AT_KEY)?; + + Ok(()) +} + +fn find_rest_embedders( + index_uid: &str, + index_env: &Env, + index_txn: &RoTxn, +) -> anyhow::Result> { + let main = try_opening_poly_database(index_env, index_txn, db_name::MAIN) + .with_context(|| format!("while checking REST embedders for index `{index_uid}`"))?; + + let mut rest_embedders = vec![]; + + for config in main + .remap_types::>>() + .get(index_txn, main_key::EMBEDDING_CONFIGS)? + .unwrap_or_default() + { + if let v1_9::EmbedderOptions::Rest(_) = config.config.embedder_options { + rest_embedders.push(config.name); + } + } + + Ok(rest_embedders) +} + +fn date_round_trip( + wtxn: &mut RwTxn, + index_uid: &str, + db: Database, + key: &str, +) -> anyhow::Result<()> { + let datetime = + db.remap_types::>().get(wtxn, key).with_context( + || format!("could not read `{key}` while updating date format for index `{index_uid}`"), + )?; + + if let Some(datetime) = datetime { + db.remap_types::>() + .put(wtxn, key, &self::OffsetDateTime(datetime.0)) + .with_context(|| { + format!( + "could not write `{key}` while updating date format for index `{index_uid}`" + ) + })?; + } + + Ok(()) +} + +pub fn v1_9_to_v1_10(db_path: &Path) -> anyhow::Result<()> { + println!("Upgrading from v1.9.0 to v1.10.0"); + // 2 changes here + + // 1. date format. needs to be done before opening the Index + // 2. REST embedders. We don't support this case right now, so bail + + let index_scheduler_path = db_path.join("tasks"); + let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&index_scheduler_path) } + .with_context(|| format!("While trying to open {:?}", index_scheduler_path.display()))?; + + let mut sched_wtxn = env.write_txn()?; + + let index_mapping: Database = + try_opening_database(&env, &sched_wtxn, "index-mapping")?; + + let index_stats: Database = + try_opening_database(&env, &sched_wtxn, "index-stats").with_context(|| { + format!("While trying to open {:?}", index_scheduler_path.display()) + })?; + + let index_count = + index_mapping.len(&sched_wtxn).context("while reading the number of indexes")?; + + // FIXME: not ideal, we have to pre-populate all indexes to prevent double borrow of sched_wtxn + // 1. immutably for the iteration + // 2. mutably for updating index stats + let indexes: Vec<_> = index_mapping + .iter(&sched_wtxn)? + .map(|res| res.map(|(uid, uuid)| (uid.to_owned(), uuid))) + .collect(); + + let mut rest_embedders = Vec::new(); + + let mut unwrapped_indexes = Vec::new(); + + // check that update can take place + for (index_index, result) in indexes.into_iter().enumerate() { + let (uid, uuid) = result?; + let index_path = db_path.join("indexes").join(uuid.to_string()); + + println!( + "[{}/{index_count}]Checking that update can take place for `{uid}` at `{}`", + index_index + 1, + index_path.display() + ); + + let index_env = unsafe { + // FIXME: fetch the 25 magic number from the index file + EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| { + format!("while opening index {uid} at '{}'", index_path.display()) + })? + }; + + let index_txn = index_env.read_txn().with_context(|| { + format!( + "while obtaining a write transaction for index {uid} at {}", + index_path.display() + ) + })?; + + println!("\t- Checking for incompatible embedders (REST embedders)"); + let rest_embedders_for_index = find_rest_embedders(&uid, &index_env, &index_txn)?; + + if rest_embedders_for_index.is_empty() { + unwrapped_indexes.push((uid, uuid)); + } else { + // no need to add to unwrapped indexes because we'll exit early + rest_embedders.push((uid, rest_embedders_for_index)); + } + } + + if !rest_embedders.is_empty() { + let rest_embedders = rest_embedders + .into_iter() + .flat_map(|(index, embedders)| std::iter::repeat(index.clone()).zip(embedders)) + .map(|(index, embedder)| format!("\t- embedder `{embedder}` in index `{index}`")) + .collect::>() + .join("\n"); + bail!("The update cannot take place because there are REST embedder(s). Remove them before proceeding with the update:\n{rest_embedders}\n\n\ + The database has not been modified and is still a valid v1.9 database."); + } + + println!("Update can take place, updating"); + + for (index_index, (uid, uuid)) in unwrapped_indexes.into_iter().enumerate() { + let index_path = db_path.join("indexes").join(uuid.to_string()); + + println!( + "[{}/{index_count}]Updating index `{uid}` at `{}`", + index_index + 1, + index_path.display() + ); + + let index_env = unsafe { + // FIXME: fetch the 25 magic number from the index file + EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| { + format!("while opening index {uid} at '{}'", index_path.display()) + })? + }; + + let mut index_wtxn = index_env.write_txn().with_context(|| { + format!( + "while obtaining a write transaction for index `{uid}` at `{}`", + index_path.display() + ) + })?; + + println!("\t- Updating index stats"); + update_index_stats(index_stats, &uid, uuid, &mut sched_wtxn)?; + println!("\t- Updating date format"); + update_date_format(&uid, &index_env, &mut index_wtxn)?; + + index_wtxn.commit().with_context(|| { + format!("while committing the write txn for index `{uid}` at {}", index_path.display()) + })?; + } + + sched_wtxn.commit().context("while committing the write txn for the index-scheduler")?; + + println!("Upgrading database succeeded"); + + Ok(()) +} diff --git a/crates/meilitool/src/upgrade/v1_11.rs b/crates/meilitool/src/upgrade/v1_11.rs new file mode 100644 index 000000000..0c84d3842 --- /dev/null +++ b/crates/meilitool/src/upgrade/v1_11.rs @@ -0,0 +1,85 @@ +//! The breaking changes that happened between the v1.10 and the v1.11 are: +//! - Arroy went from the v0.4.0 to the v0.5.0, see this release note to get the whole context: https://github.com/meilisearch/arroy/releases/tag/v0.5.0 +//! - The `angular` distance has been renamed to `cosine` => We only need to update the string in the metadata. +//! - Reorganize the `NodeId` to make the appending of vectors work => We'll have to update the keys of almost all items in the DB. +//! - Store the list of updated IDs directly in LMDB instead of a roaring bitmap => This shouldn't be an issue since we are never supposed to commit this roaring bitmap, but it's not forbidden by arroy so ensuring it works is probably better than anything. + +use std::path::Path; + +use anyhow::Context; +use meilisearch_types::{ + heed::{types::Str, Database, EnvOpenOptions}, + milli::index::db_name, +}; + +use crate::{try_opening_database, try_opening_poly_database, uuid_codec::UuidCodec}; + +pub fn v1_10_to_v1_11(db_path: &Path) -> anyhow::Result<()> { + println!("Upgrading from v1.10.0 to v1.11.0"); + + let index_scheduler_path = db_path.join("tasks"); + let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&index_scheduler_path) } + .with_context(|| format!("While trying to open {:?}", index_scheduler_path.display()))?; + + let sched_rtxn = env.read_txn()?; + + let index_mapping: Database = + try_opening_database(&env, &sched_rtxn, "index-mapping")?; + + let index_count = + index_mapping.len(&sched_rtxn).context("while reading the number of indexes")?; + + let indexes: Vec<_> = index_mapping + .iter(&sched_rtxn)? + .map(|res| res.map(|(uid, uuid)| (uid.to_owned(), uuid))) + .collect(); + + for (index_index, result) in indexes.into_iter().enumerate() { + let (uid, uuid) = result?; + let index_path = db_path.join("indexes").join(uuid.to_string()); + + println!( + "[{}/{index_count}]Updating embeddings for `{uid}` at `{}`", + index_index + 1, + index_path.display() + ); + + let index_env = unsafe { + EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| { + format!("while opening index {uid} at '{}'", index_path.display()) + })? + }; + + let index_rtxn = index_env.read_txn().with_context(|| { + format!( + "while obtaining a read transaction for index {uid} at {}", + index_path.display() + ) + })?; + let index_read_database = + try_opening_poly_database(&index_env, &index_rtxn, db_name::VECTOR_ARROY) + .with_context(|| format!("while updating date format for index `{uid}`"))?; + + let mut index_wtxn = index_env.write_txn().with_context(|| { + format!( + "while obtaining a write transaction for index {uid} at {}", + index_path.display() + ) + })?; + + let index_write_database = + try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_ARROY) + .with_context(|| format!("while updating date format for index `{uid}`"))?; + + arroy_v04_to_v05::ugrade_from_prev_version( + &index_rtxn, + index_read_database, + &mut index_wtxn, + index_write_database, + )?; + + index_wtxn.commit()?; + } + + Ok(()) +} diff --git a/crates/meilitool/src/upgrade/v1_9.rs b/crates/meilitool/src/upgrade/v1_9.rs new file mode 100644 index 000000000..96cbfe68c --- /dev/null +++ b/crates/meilitool/src/upgrade/v1_9.rs @@ -0,0 +1,158 @@ +use serde::{Deserialize, Serialize}; +use time::{Date, OffsetDateTime, Time, UtcOffset}; + +pub type FieldDistribution = std::collections::BTreeMap; + +/// The statistics that can be computed from an `Index` object. +#[derive(serde::Deserialize, Debug)] +pub struct IndexStats { + /// Number of documents in the index. + pub number_of_documents: u64, + /// Size taken up by the index' DB, in bytes. + /// + /// This includes the size taken by both the used and free pages of the DB, and as the free pages + /// are not returned to the disk after a deletion, this number is typically larger than + /// `used_database_size` that only includes the size of the used pages. + pub database_size: u64, + /// Size taken by the used pages of the index' DB, in bytes. + /// + /// As the DB backend does not return to the disk the pages that are not currently used by the DB, + /// this value is typically smaller than `database_size`. + pub used_database_size: u64, + /// Association of every field name with the number of times it occurs in the documents. + pub field_distribution: FieldDistribution, + /// Creation date of the index. + pub created_at: LegacyDateTime, + /// Date of the last update of the index. + pub updated_at: LegacyDateTime, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct IndexEmbeddingConfig { + pub name: String, + pub config: EmbeddingConfig, +} + +#[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)] +pub struct EmbeddingConfig { + /// Options of the embedder, specific to each kind of embedder + pub embedder_options: EmbedderOptions, +} + +/// Options of an embedder, specific to each kind of embedder. +#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +pub enum EmbedderOptions { + HuggingFace(hf::EmbedderOptions), + OpenAi(openai::EmbedderOptions), + Ollama(ollama::EmbedderOptions), + UserProvided(manual::EmbedderOptions), + Rest(rest::EmbedderOptions), +} + +impl Default for EmbedderOptions { + fn default() -> Self { + Self::OpenAi(openai::EmbedderOptions { api_key: None, dimensions: None }) + } +} + +mod hf { + #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] + pub struct EmbedderOptions { + pub model: String, + pub revision: Option, + } +} +mod openai { + + #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] + pub struct EmbedderOptions { + pub api_key: Option, + pub dimensions: Option, + } +} +mod ollama { + #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] + pub struct EmbedderOptions { + pub embedding_model: String, + pub url: Option, + pub api_key: Option, + } +} +mod manual { + #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] + pub struct EmbedderOptions { + pub dimensions: usize, + } +} +mod rest { + #[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize, Hash)] + pub struct EmbedderOptions { + pub api_key: Option, + pub dimensions: Option, + pub url: String, + pub input_field: Vec, + // path to the array of embeddings + pub path_to_embeddings: Vec, + // shape of a single embedding + pub embedding_object: Vec, + } +} + +/// A datetime from Meilisearch v1.9 with an unspecified format. +#[derive(Debug)] +pub struct LegacyDateTime(pub OffsetDateTime); + +impl<'de> Deserialize<'de> for LegacyDateTime { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = OffsetDateTime; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a valid datetime") + } + + // Comes from a binary. The legacy format is: + // 2024-11-04 13:32:08.48368 +00:00:00 + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + let format = time::macros::format_description!("[year]-[month]-[day] [hour]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]"); + OffsetDateTime::parse(v, format).map_err(E::custom) + } + + // Comes from the docker image, the legacy format is: + // [2024, 309, 17, 15, 1, 698184971, 0,0,0] + // year, day in year, hour, minute, sec, subsec , offset stuff + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut vec = Vec::new(); + // We must deserialize the value as `i64` because the largest values are `u32` and `i32` + while let Some(el) = seq.next_element::()? { + vec.push(el); + } + if vec.len() != 9 { + return Err(serde::de::Error::custom(format!( + "Invalid datetime, received an array of {} elements instead of 9", + vec.len() + ))); + } + Ok(OffsetDateTime::new_in_offset( + Date::from_ordinal_date(vec[0] as i32, vec[1] as u16) + .map_err(serde::de::Error::custom)?, + Time::from_hms_nano(vec[2] as u8, vec[3] as u8, vec[4] as u8, vec[5] as u32) + .map_err(serde::de::Error::custom)?, + UtcOffset::from_hms(vec[6] as i8, vec[7] as i8, vec[8] as i8) + .map_err(serde::de::Error::custom)?, + )) + } + } + deserializer.deserialize_any(Visitor).map(LegacyDateTime) + } +} diff --git a/meilitool/src/uuid_codec.rs b/crates/meilitool/src/uuid_codec.rs similarity index 100% rename from meilitool/src/uuid_codec.rs rename to crates/meilitool/src/uuid_codec.rs diff --git a/milli/Cargo.toml b/crates/milli/Cargo.toml similarity index 91% rename from milli/Cargo.toml rename to crates/milli/Cargo.toml index df463c902..005393411 100644 --- a/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -16,7 +16,7 @@ big_s = "1.0.2" bimap = { version = "0.6.3", features = ["serde"] } bincode = "1.3.3" bstr = "1.9.1" -bytemuck = { version = "1.16.1", features = ["extern_crate_alloc"] } +bytemuck = { version = "1.18.0", features = ["extern_crate_alloc"] } byteorder = "1.5.0" # charabia = { version = "0.9.0", default-features = false } charabia = { git = "https://github.com/meilisearch/charabia", branch = "mutualize-char-normalizer", default-features = false } @@ -88,7 +88,7 @@ rhai = { git = "https://github.com/rhaiscript/rhai", rev = "ef3df63121d27aacd838 "no_time", "sync", ] } -arroy = { git = "https://github.com/meilisearch/arroy/", rev = "2386594dfb009ce08821a925ccc89fb8e30bf73d" } +arroy = "0.5.0" rand = "0.8.5" tracing = "0.1.40" ureq = { version = "2.10.0", features = ["json"] } @@ -111,16 +111,7 @@ rand = { version = "0.8.5", features = ["small_rng"] } [features] all-tokenizations = [ - "charabia/chinese", - "charabia/hebrew", - "charabia/japanese", - "charabia/thai", - "charabia/korean", - "charabia/greek", - "charabia/khmer", - "charabia/vietnamese", - "charabia/swedish-recomposition", - "charabia/german-segmentation", + "charabia/default", ] # Use POSIX semaphores instead of SysV semaphores in LMDB @@ -159,5 +150,8 @@ german = ["charabia/german-segmentation"] # force swedish character recomposition swedish-recomposition = ["charabia/swedish-recomposition"] +# allow turkish specialized tokenization +turkish = ["charabia/turkish"] + # allow CUDA support, see cuda = ["candle-core/cuda"] diff --git a/milli/README.md b/crates/milli/README.md similarity index 100% rename from milli/README.md rename to crates/milli/README.md diff --git a/milli/examples/index.rs b/crates/milli/examples/index.rs similarity index 100% rename from milli/examples/index.rs rename to crates/milli/examples/index.rs diff --git a/milli/examples/search.rs b/crates/milli/examples/search.rs similarity index 100% rename from milli/examples/search.rs rename to crates/milli/examples/search.rs diff --git a/milli/examples/settings.rs b/crates/milli/examples/settings.rs similarity index 100% rename from milli/examples/settings.rs rename to crates/milli/examples/settings.rs diff --git a/milli/fuzz/.gitignore b/crates/milli/fuzz/.gitignore similarity index 100% rename from milli/fuzz/.gitignore rename to crates/milli/fuzz/.gitignore diff --git a/milli/src/asc_desc.rs b/crates/milli/src/asc_desc.rs similarity index 100% rename from milli/src/asc_desc.rs rename to crates/milli/src/asc_desc.rs diff --git a/milli/src/criterion.rs b/crates/milli/src/criterion.rs similarity index 100% rename from milli/src/criterion.rs rename to crates/milli/src/criterion.rs diff --git a/milli/src/documents/builder.rs b/crates/milli/src/documents/builder.rs similarity index 100% rename from milli/src/documents/builder.rs rename to crates/milli/src/documents/builder.rs diff --git a/milli/src/documents/enriched.rs b/crates/milli/src/documents/enriched.rs similarity index 100% rename from milli/src/documents/enriched.rs rename to crates/milli/src/documents/enriched.rs diff --git a/milli/src/documents/mod.rs b/crates/milli/src/documents/mod.rs similarity index 100% rename from milli/src/documents/mod.rs rename to crates/milli/src/documents/mod.rs diff --git a/milli/src/documents/primary_key.rs b/crates/milli/src/documents/primary_key.rs similarity index 98% rename from milli/src/documents/primary_key.rs rename to crates/milli/src/documents/primary_key.rs index 79fd07048..91d15de6d 100644 --- a/milli/src/documents/primary_key.rs +++ b/crates/milli/src/documents/primary_key.rs @@ -319,12 +319,13 @@ fn starts_with(selector: &str, key: &str) -> bool { // FIXME: move to a DocumentId struct pub fn validate_document_id_str(document_id: &str) -> Option<&str> { - if !document_id.is_empty() - && document_id.chars().all(|c| matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_')) + if document_id.is_empty() + || document_id.len() > 512 + || !document_id.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') { - Some(document_id) - } else { None + } else { + Some(document_id) } } @@ -335,6 +336,7 @@ pub fn validate_document_id_value(document_id: Value) -> StdResult Ok(s.to_string()), None => Err(UserError::InvalidDocumentId { document_id: Value::String(string) }), }, + // a `u64` or `i64` cannot be more than 512 bytes once converted to a string Value::Number(number) if !number.is_f64() => Ok(number.to_string()), content => Err(UserError::InvalidDocumentId { document_id: content }), } diff --git a/milli/src/documents/reader.rs b/crates/milli/src/documents/reader.rs similarity index 100% rename from milli/src/documents/reader.rs rename to crates/milli/src/documents/reader.rs diff --git a/milli/src/documents/serde_impl.rs b/crates/milli/src/documents/serde_impl.rs similarity index 100% rename from milli/src/documents/serde_impl.rs rename to crates/milli/src/documents/serde_impl.rs diff --git a/milli/src/error.rs b/crates/milli/src/error.rs similarity index 99% rename from milli/src/error.rs rename to crates/milli/src/error.rs index e6856c4ef..53d9827ac 100644 --- a/milli/src/error.rs +++ b/crates/milli/src/error.rs @@ -106,7 +106,8 @@ pub enum UserError { #[error( "Document identifier `{}` is invalid. \ A document identifier can be of type integer or string, \ -only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_).", .document_id.to_string() +only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_), \ +and can not be more than 512 bytes.", .document_id.to_string() )] InvalidDocumentId { document_id: Value }, #[error("Invalid facet distribution, {}", format_invalid_filter_distribution(.invalid_facets_name, .valid_facets_name))] @@ -297,6 +298,7 @@ impl From for Error { arroy::Error::InvalidVecDimension { expected, received } => { Error::UserError(UserError::InvalidVectorDimensions { expected, found: received }) } + arroy::Error::BuildCancelled => Error::InternalError(InternalError::AbortedIndexation), arroy::Error::DatabaseFull | arroy::Error::InvalidItemAppend | arroy::Error::UnmatchingDistance { .. } diff --git a/milli/src/external_documents_ids.rs b/crates/milli/src/external_documents_ids.rs similarity index 100% rename from milli/src/external_documents_ids.rs rename to crates/milli/src/external_documents_ids.rs diff --git a/milli/src/facet/facet_type.rs b/crates/milli/src/facet/facet_type.rs similarity index 100% rename from milli/src/facet/facet_type.rs rename to crates/milli/src/facet/facet_type.rs diff --git a/milli/src/facet/facet_value.rs b/crates/milli/src/facet/facet_value.rs similarity index 100% rename from milli/src/facet/facet_value.rs rename to crates/milli/src/facet/facet_value.rs diff --git a/milli/src/facet/mod.rs b/crates/milli/src/facet/mod.rs similarity index 100% rename from milli/src/facet/mod.rs rename to crates/milli/src/facet/mod.rs diff --git a/milli/src/facet/value_encoding.rs b/crates/milli/src/facet/value_encoding.rs similarity index 100% rename from milli/src/facet/value_encoding.rs rename to crates/milli/src/facet/value_encoding.rs diff --git a/milli/src/fieldids_weights_map.rs b/crates/milli/src/fieldids_weights_map.rs similarity index 100% rename from milli/src/fieldids_weights_map.rs rename to crates/milli/src/fieldids_weights_map.rs diff --git a/milli/src/fields_ids_map.rs b/crates/milli/src/fields_ids_map.rs similarity index 100% rename from milli/src/fields_ids_map.rs rename to crates/milli/src/fields_ids_map.rs diff --git a/milli/src/fields_ids_map/global.rs b/crates/milli/src/fields_ids_map/global.rs similarity index 100% rename from milli/src/fields_ids_map/global.rs rename to crates/milli/src/fields_ids_map/global.rs diff --git a/milli/src/heed_codec/beu16_str_codec.rs b/crates/milli/src/heed_codec/beu16_str_codec.rs similarity index 100% rename from milli/src/heed_codec/beu16_str_codec.rs rename to crates/milli/src/heed_codec/beu16_str_codec.rs diff --git a/milli/src/heed_codec/beu32_str_codec.rs b/crates/milli/src/heed_codec/beu32_str_codec.rs similarity index 100% rename from milli/src/heed_codec/beu32_str_codec.rs rename to crates/milli/src/heed_codec/beu32_str_codec.rs diff --git a/milli/src/heed_codec/byte_slice_ref.rs b/crates/milli/src/heed_codec/byte_slice_ref.rs similarity index 100% rename from milli/src/heed_codec/byte_slice_ref.rs rename to crates/milli/src/heed_codec/byte_slice_ref.rs diff --git a/milli/src/heed_codec/facet/field_doc_id_facet_codec.rs b/crates/milli/src/heed_codec/facet/field_doc_id_facet_codec.rs similarity index 100% rename from milli/src/heed_codec/facet/field_doc_id_facet_codec.rs rename to crates/milli/src/heed_codec/facet/field_doc_id_facet_codec.rs diff --git a/milli/src/heed_codec/facet/mod.rs b/crates/milli/src/heed_codec/facet/mod.rs similarity index 100% rename from milli/src/heed_codec/facet/mod.rs rename to crates/milli/src/heed_codec/facet/mod.rs diff --git a/milli/src/heed_codec/facet/ordered_f64_codec.rs b/crates/milli/src/heed_codec/facet/ordered_f64_codec.rs similarity index 100% rename from milli/src/heed_codec/facet/ordered_f64_codec.rs rename to crates/milli/src/heed_codec/facet/ordered_f64_codec.rs diff --git a/milli/src/heed_codec/field_id_word_count_codec.rs b/crates/milli/src/heed_codec/field_id_word_count_codec.rs similarity index 100% rename from milli/src/heed_codec/field_id_word_count_codec.rs rename to crates/milli/src/heed_codec/field_id_word_count_codec.rs diff --git a/milli/src/heed_codec/fst_set_codec.rs b/crates/milli/src/heed_codec/fst_set_codec.rs similarity index 100% rename from milli/src/heed_codec/fst_set_codec.rs rename to crates/milli/src/heed_codec/fst_set_codec.rs diff --git a/milli/src/heed_codec/mod.rs b/crates/milli/src/heed_codec/mod.rs similarity index 100% rename from milli/src/heed_codec/mod.rs rename to crates/milli/src/heed_codec/mod.rs diff --git a/milli/src/heed_codec/obkv_codec.rs b/crates/milli/src/heed_codec/obkv_codec.rs similarity index 100% rename from milli/src/heed_codec/obkv_codec.rs rename to crates/milli/src/heed_codec/obkv_codec.rs diff --git a/milli/src/heed_codec/roaring_bitmap/bo_roaring_bitmap_codec.rs b/crates/milli/src/heed_codec/roaring_bitmap/bo_roaring_bitmap_codec.rs similarity index 100% rename from milli/src/heed_codec/roaring_bitmap/bo_roaring_bitmap_codec.rs rename to crates/milli/src/heed_codec/roaring_bitmap/bo_roaring_bitmap_codec.rs diff --git a/milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs b/crates/milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs similarity index 100% rename from milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs rename to crates/milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs diff --git a/milli/src/heed_codec/roaring_bitmap/mod.rs b/crates/milli/src/heed_codec/roaring_bitmap/mod.rs similarity index 100% rename from milli/src/heed_codec/roaring_bitmap/mod.rs rename to crates/milli/src/heed_codec/roaring_bitmap/mod.rs diff --git a/milli/src/heed_codec/roaring_bitmap/roaring_bitmap_codec.rs b/crates/milli/src/heed_codec/roaring_bitmap/roaring_bitmap_codec.rs similarity index 100% rename from milli/src/heed_codec/roaring_bitmap/roaring_bitmap_codec.rs rename to crates/milli/src/heed_codec/roaring_bitmap/roaring_bitmap_codec.rs diff --git a/milli/src/heed_codec/roaring_bitmap_length/bo_roaring_bitmap_len_codec.rs b/crates/milli/src/heed_codec/roaring_bitmap_length/bo_roaring_bitmap_len_codec.rs similarity index 100% rename from milli/src/heed_codec/roaring_bitmap_length/bo_roaring_bitmap_len_codec.rs rename to crates/milli/src/heed_codec/roaring_bitmap_length/bo_roaring_bitmap_len_codec.rs diff --git a/milli/src/heed_codec/roaring_bitmap_length/cbo_roaring_bitmap_len_codec.rs b/crates/milli/src/heed_codec/roaring_bitmap_length/cbo_roaring_bitmap_len_codec.rs similarity index 100% rename from milli/src/heed_codec/roaring_bitmap_length/cbo_roaring_bitmap_len_codec.rs rename to crates/milli/src/heed_codec/roaring_bitmap_length/cbo_roaring_bitmap_len_codec.rs diff --git a/milli/src/heed_codec/roaring_bitmap_length/mod.rs b/crates/milli/src/heed_codec/roaring_bitmap_length/mod.rs similarity index 100% rename from milli/src/heed_codec/roaring_bitmap_length/mod.rs rename to crates/milli/src/heed_codec/roaring_bitmap_length/mod.rs diff --git a/milli/src/heed_codec/roaring_bitmap_length/roaring_bitmap_len_codec.rs b/crates/milli/src/heed_codec/roaring_bitmap_length/roaring_bitmap_len_codec.rs similarity index 100% rename from milli/src/heed_codec/roaring_bitmap_length/roaring_bitmap_len_codec.rs rename to crates/milli/src/heed_codec/roaring_bitmap_length/roaring_bitmap_len_codec.rs diff --git a/milli/src/heed_codec/str_beu32_codec.rs b/crates/milli/src/heed_codec/str_beu32_codec.rs similarity index 100% rename from milli/src/heed_codec/str_beu32_codec.rs rename to crates/milli/src/heed_codec/str_beu32_codec.rs diff --git a/milli/src/heed_codec/str_ref.rs b/crates/milli/src/heed_codec/str_ref.rs similarity index 100% rename from milli/src/heed_codec/str_ref.rs rename to crates/milli/src/heed_codec/str_ref.rs diff --git a/milli/src/heed_codec/str_str_u8_codec.rs b/crates/milli/src/heed_codec/str_str_u8_codec.rs similarity index 100% rename from milli/src/heed_codec/str_str_u8_codec.rs rename to crates/milli/src/heed_codec/str_str_u8_codec.rs diff --git a/milli/src/index.rs b/crates/milli/src/index.rs similarity index 98% rename from milli/src/index.rs rename to crates/milli/src/index.rs index 19064e8d7..08a8e36f8 100644 --- a/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1618,24 +1618,6 @@ impl Index { .unwrap_or_default()) } - pub fn arroy_readers<'a>( - &'a self, - rtxn: &'a RoTxn<'a>, - embedder_id: u8, - quantized: bool, - ) -> impl Iterator> + 'a { - crate::vector::arroy_db_range_for_embedder(embedder_id).map_while(move |k| { - let reader = ArroyWrapper::new(self.vector_arroy, k, quantized); - // Here we don't care about the dimensions, but we want to know if we can read - // in the database or if its metadata are missing because there is no document with that many vectors. - match reader.dimensions(rtxn) { - Ok(_) => Some(Ok(reader)), - Err(arroy::Error::MissingMetadata(_)) => None, - Err(e) => Some(Err(e.into())), - } - }) - } - pub(crate) fn put_search_cutoff(&self, wtxn: &mut RwTxn<'_>, cutoff: u64) -> heed::Result<()> { self.main.remap_types::().put(wtxn, main_key::SEARCH_CUTOFF, &cutoff) } @@ -1657,14 +1639,9 @@ impl Index { let embedding_configs = self.embedding_configs(rtxn)?; for config in embedding_configs { let embedder_id = self.embedder_category_id.get(rtxn, &config.name)?.unwrap(); - let embeddings = self - .arroy_readers(rtxn, embedder_id, config.config.quantized()) - .map_while(|reader| { - reader - .and_then(|r| r.item_vector(rtxn, docid).map_err(|e| e.into())) - .transpose() - }) - .collect::>>()?; + let reader = + ArroyWrapper::new(self.vector_arroy, embedder_id, config.config.quantized()); + let embeddings = reader.item_vectors(rtxn, docid)?; res.insert(config.name.to_owned(), embeddings); } Ok(res) diff --git a/milli/src/lib.rs b/crates/milli/src/lib.rs similarity index 100% rename from milli/src/lib.rs rename to crates/milli/src/lib.rs diff --git a/milli/src/localized_attributes_rules.rs b/crates/milli/src/localized_attributes_rules.rs similarity index 100% rename from milli/src/localized_attributes_rules.rs rename to crates/milli/src/localized_attributes_rules.rs diff --git a/milli/src/order_by_map.rs b/crates/milli/src/order_by_map.rs similarity index 100% rename from milli/src/order_by_map.rs rename to crates/milli/src/order_by_map.rs diff --git a/milli/src/prompt/context.rs b/crates/milli/src/prompt/context.rs similarity index 100% rename from milli/src/prompt/context.rs rename to crates/milli/src/prompt/context.rs diff --git a/milli/src/prompt/document.rs b/crates/milli/src/prompt/document.rs similarity index 100% rename from milli/src/prompt/document.rs rename to crates/milli/src/prompt/document.rs diff --git a/milli/src/prompt/error.rs b/crates/milli/src/prompt/error.rs similarity index 100% rename from milli/src/prompt/error.rs rename to crates/milli/src/prompt/error.rs diff --git a/milli/src/prompt/fields.rs b/crates/milli/src/prompt/fields.rs similarity index 100% rename from milli/src/prompt/fields.rs rename to crates/milli/src/prompt/fields.rs diff --git a/milli/src/prompt/mod.rs b/crates/milli/src/prompt/mod.rs similarity index 100% rename from milli/src/prompt/mod.rs rename to crates/milli/src/prompt/mod.rs diff --git a/milli/src/prompt/template_checker.rs b/crates/milli/src/prompt/template_checker.rs similarity index 100% rename from milli/src/prompt/template_checker.rs rename to crates/milli/src/prompt/template_checker.rs diff --git a/milli/src/proximity.rs b/crates/milli/src/proximity.rs similarity index 100% rename from milli/src/proximity.rs rename to crates/milli/src/proximity.rs diff --git a/milli/src/score_details.rs b/crates/milli/src/score_details.rs similarity index 100% rename from milli/src/score_details.rs rename to crates/milli/src/score_details.rs diff --git a/milli/src/search/facet/facet_distribution.rs b/crates/milli/src/search/facet/facet_distribution.rs similarity index 100% rename from milli/src/search/facet/facet_distribution.rs rename to crates/milli/src/search/facet/facet_distribution.rs diff --git a/milli/src/search/facet/facet_distribution_iter.rs b/crates/milli/src/search/facet/facet_distribution_iter.rs similarity index 100% rename from milli/src/search/facet/facet_distribution_iter.rs rename to crates/milli/src/search/facet/facet_distribution_iter.rs diff --git a/milli/src/search/facet/facet_range_search.rs b/crates/milli/src/search/facet/facet_range_search.rs similarity index 100% rename from milli/src/search/facet/facet_range_search.rs rename to crates/milli/src/search/facet/facet_range_search.rs diff --git a/milli/src/search/facet/facet_sort_ascending.rs b/crates/milli/src/search/facet/facet_sort_ascending.rs similarity index 100% rename from milli/src/search/facet/facet_sort_ascending.rs rename to crates/milli/src/search/facet/facet_sort_ascending.rs diff --git a/milli/src/search/facet/facet_sort_descending.rs b/crates/milli/src/search/facet/facet_sort_descending.rs similarity index 100% rename from milli/src/search/facet/facet_sort_descending.rs rename to crates/milli/src/search/facet/facet_sort_descending.rs diff --git a/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs similarity index 100% rename from milli/src/search/facet/filter.rs rename to crates/milli/src/search/facet/filter.rs diff --git a/milli/src/search/facet/mod.rs b/crates/milli/src/search/facet/mod.rs similarity index 100% rename from milli/src/search/facet/mod.rs rename to crates/milli/src/search/facet/mod.rs diff --git a/milli/src/search/facet/search.rs b/crates/milli/src/search/facet/search.rs similarity index 100% rename from milli/src/search/facet/search.rs rename to crates/milli/src/search/facet/search.rs diff --git a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/0.snap b/crates/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/0.snap rename to crates/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/0.snap diff --git a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/1.snap b/crates/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/1.snap rename to crates/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all/1.snap diff --git a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/0.snap b/crates/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/0.snap rename to crates/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/0.snap diff --git a/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/1.snap b/crates/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/1.snap rename to crates/milli/src/search/facet/snapshots/facet_distribution_iter.rs/filter_distribution_all_stop_early/1.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_0.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_0.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_0.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_0.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_1.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_1.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_1.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_1.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/excluded_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_0.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_0.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_0.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_0.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_1.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_1.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_1.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_1.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_decreasing/included_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_0.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_0.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_0.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_0.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_1.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_1.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_1.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_1.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_0_exact_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_0.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_0.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_0.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_1.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_1.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_1.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_exact/field_id_1_exact_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_0.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_0.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_0.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_0.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_1.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_1.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_1.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_1.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/excluded_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_0.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_0.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_0.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_0.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_1.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_1.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_1.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_1.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_increasing/included_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_0.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_0.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_0.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_0.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_1.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_1.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_1.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_1.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/excluded_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_0.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_0.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_0.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_0.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_1.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_1.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_1.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_1.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_pinch/included_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_0.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_0.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_0.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_0.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_1.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_1.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_1.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_1.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/end_at_included_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_0.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_0.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_0.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_0.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_1.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_1.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_1.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_1.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_2.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_2.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_2.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_2.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_3.hash.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_3.hash.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_3.hash.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/start_from_included_3.hash.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_0.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_0.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_0.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_1.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_1.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_1.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_2.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_2.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_2.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_2.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_3.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_3.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_3.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_0_3.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_0.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_0.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_0.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_1.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_1.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_1.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_2.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_2.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_2.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_2.snap diff --git a/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_3.snap b/crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_3.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_3.snap rename to crates/milli/src/search/facet/snapshots/facet_range_search.rs/filter_range_unbounded/unbounded_field_id_1_3.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending/0.snap b/crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending/0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending/0.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending/0.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending/1.snap b/crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending/1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending/1.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending/1.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/0-0.snap b/crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/0-0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/0-0.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/0-0.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/0-1.snap b/crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/0-1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/0-1.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/0-1.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/1-0.snap b/crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/1-0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/1-0.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/1-0.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/1-1.snap b/crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/1-1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/1-1.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_ascending.rs/filter_sort_ascending_multiple_field_ids/1-1.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/0.snap b/crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/0.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/0.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/1.snap b/crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/1.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/1.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/2.snap b/crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/2.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/2.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending/2.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/0-0.snap b/crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/0-0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/0-0.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/0-0.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/0-1.snap b/crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/0-1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/0-1.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/0-1.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/1-0.snap b/crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/1-0.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/1-0.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/1-0.snap diff --git a/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/1-1.snap b/crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/1-1.snap similarity index 100% rename from milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/1-1.snap rename to crates/milli/src/search/facet/snapshots/facet_sort_descending.rs/filter_sort_descending_multiple_field_ids/1-1.snap diff --git a/milli/src/search/fst_utils.rs b/crates/milli/src/search/fst_utils.rs similarity index 100% rename from milli/src/search/fst_utils.rs rename to crates/milli/src/search/fst_utils.rs diff --git a/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs similarity index 100% rename from milli/src/search/hybrid.rs rename to crates/milli/src/search/hybrid.rs diff --git a/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs similarity index 100% rename from milli/src/search/mod.rs rename to crates/milli/src/search/mod.rs diff --git a/milli/src/search/new/bucket_sort.rs b/crates/milli/src/search/new/bucket_sort.rs similarity index 100% rename from milli/src/search/new/bucket_sort.rs rename to crates/milli/src/search/new/bucket_sort.rs diff --git a/milli/src/search/new/db_cache.rs b/crates/milli/src/search/new/db_cache.rs similarity index 100% rename from milli/src/search/new/db_cache.rs rename to crates/milli/src/search/new/db_cache.rs diff --git a/milli/src/search/new/distinct.rs b/crates/milli/src/search/new/distinct.rs similarity index 100% rename from milli/src/search/new/distinct.rs rename to crates/milli/src/search/new/distinct.rs diff --git a/milli/src/search/new/exact_attribute.rs b/crates/milli/src/search/new/exact_attribute.rs similarity index 100% rename from milli/src/search/new/exact_attribute.rs rename to crates/milli/src/search/new/exact_attribute.rs diff --git a/milli/src/search/new/geo_sort.rs b/crates/milli/src/search/new/geo_sort.rs similarity index 100% rename from milli/src/search/new/geo_sort.rs rename to crates/milli/src/search/new/geo_sort.rs diff --git a/milli/src/search/new/graph_based_ranking_rule.rs b/crates/milli/src/search/new/graph_based_ranking_rule.rs similarity index 100% rename from milli/src/search/new/graph_based_ranking_rule.rs rename to crates/milli/src/search/new/graph_based_ranking_rule.rs diff --git a/milli/src/search/new/interner.rs b/crates/milli/src/search/new/interner.rs similarity index 100% rename from milli/src/search/new/interner.rs rename to crates/milli/src/search/new/interner.rs diff --git a/milli/src/search/new/limits.rs b/crates/milli/src/search/new/limits.rs similarity index 100% rename from milli/src/search/new/limits.rs rename to crates/milli/src/search/new/limits.rs diff --git a/milli/src/search/new/logger/mod.rs b/crates/milli/src/search/new/logger/mod.rs similarity index 100% rename from milli/src/search/new/logger/mod.rs rename to crates/milli/src/search/new/logger/mod.rs diff --git a/milli/src/search/new/logger/visual.rs b/crates/milli/src/search/new/logger/visual.rs similarity index 100% rename from milli/src/search/new/logger/visual.rs rename to crates/milli/src/search/new/logger/visual.rs diff --git a/crates/milli/src/search/new/matches/best_match_interval.rs b/crates/milli/src/search/new/matches/best_match_interval.rs new file mode 100644 index 000000000..a6497f351 --- /dev/null +++ b/crates/milli/src/search/new/matches/best_match_interval.rs @@ -0,0 +1,139 @@ +use super::matching_words::WordId; +use super::{Match, MatchPosition}; + +struct MatchIntervalWithScore { + interval: [usize; 2], + score: [i16; 3], +} + +// count score for phrases +fn tally_phrase_scores(fwp: &usize, lwp: &usize, order_score: &mut i16, distance_score: &mut i16) { + let words_in_phrase_minus_one = (lwp - fwp) as i16; + // will always be ordered, so +1 for each space between words + *order_score += words_in_phrase_minus_one; + // distance will always be 1, so -1 for each space between words + *distance_score -= words_in_phrase_minus_one; +} + +/// Compute the score of a match interval: +/// 1) count unique matches +/// 2) calculate distance between matches +/// 3) count ordered matches +fn get_interval_score(matches: &[Match]) -> [i16; 3] { + let mut ids: Vec = Vec::with_capacity(matches.len()); + let mut order_score = 0; + let mut distance_score = 0; + + let mut iter = matches.iter().peekable(); + while let Some(m) = iter.next() { + if let Some(next_match) = iter.peek() { + // if matches are ordered + if next_match.ids.iter().min() > m.ids.iter().min() { + order_score += 1; + } + + let m_last_word_pos = match m.position { + MatchPosition::Word { word_position, .. } => word_position, + MatchPosition::Phrase { word_positions: [fwp, lwp], .. } => { + tally_phrase_scores(&fwp, &lwp, &mut order_score, &mut distance_score); + lwp + } + }; + let next_match_first_word_pos = next_match.get_first_word_pos(); + + // compute distance between matches + distance_score -= (next_match_first_word_pos - m_last_word_pos).min(7) as i16; + } else if let MatchPosition::Phrase { word_positions: [fwp, lwp], .. } = m.position { + // in case last match is a phrase, count score for its words + tally_phrase_scores(&fwp, &lwp, &mut order_score, &mut distance_score); + } + + ids.extend(m.ids.iter()); + } + + ids.sort_unstable(); + ids.dedup(); + let uniq_score = ids.len() as i16; + + // rank by unique match count, then by distance between matches, then by ordered match count. + [uniq_score, distance_score, order_score] +} + +/// Returns the first and last match where the score computed by match_interval_score is the best. +pub fn find_best_match_interval(matches: &[Match], crop_size: usize) -> [&Match; 2] { + if matches.is_empty() { + panic!("`matches` should not be empty at this point"); + } + + // positions of the first and the last match of the best matches interval in `matches`. + let mut best_interval: Option = None; + + let mut save_best_interval = |interval_first, interval_last| { + let interval_score = get_interval_score(&matches[interval_first..=interval_last]); + let is_interval_score_better = &best_interval + .as_ref() + .map_or(true, |MatchIntervalWithScore { score, .. }| interval_score > *score); + + if *is_interval_score_better { + best_interval = Some(MatchIntervalWithScore { + interval: [interval_first, interval_last], + score: interval_score, + }); + } + }; + + // we compute the matches interval if we have at least 2 matches. + // current interval positions. + let mut interval_first = 0; + let mut interval_first_match_first_word_pos = matches[interval_first].get_first_word_pos(); + + for (index, next_match) in matches.iter().enumerate() { + // if next match would make interval gross more than crop_size, + // we compare the current interval with the best one, + // then we increase `interval_first` until next match can be added. + let next_match_last_word_pos = next_match.get_last_word_pos(); + + // if the next match would mean that we pass the crop size window, + // we take the last valid match, that didn't pass this boundry, which is `index` - 1, + // and calculate a score for it, and check if it's better than our best so far + if next_match_last_word_pos - interval_first_match_first_word_pos >= crop_size { + // if index is 0 there is no last viable match + if index != 0 { + let interval_last = index - 1; + // keep interval if it's the best + save_best_interval(interval_first, interval_last); + } + + // advance start of the interval while interval is longer than crop_size. + loop { + interval_first += 1; + if interval_first == matches.len() { + interval_first -= 1; + break; + } + + interval_first_match_first_word_pos = matches[interval_first].get_first_word_pos(); + + if interval_first_match_first_word_pos > next_match_last_word_pos + || next_match_last_word_pos - interval_first_match_first_word_pos < crop_size + { + break; + } + } + } + } + + // compute the last interval score and compare it to the best one. + let interval_last = matches.len() - 1; + // if it's the last match with itself, we need to make sure it's + // not a phrase longer than the crop window + if interval_first != interval_last || matches[interval_first].get_word_count() < crop_size { + save_best_interval(interval_first, interval_last); + } + + // if none of the matches fit the criteria above, default to the first one + best_interval.map_or( + [&matches[0], &matches[0]], + |MatchIntervalWithScore { interval: [first, last], .. }| [&matches[first], &matches[last]], + ) +} diff --git a/crates/milli/src/search/new/matches/match.rs b/crates/milli/src/search/new/matches/match.rs new file mode 100644 index 000000000..2eef4d5a6 --- /dev/null +++ b/crates/milli/src/search/new/matches/match.rs @@ -0,0 +1,62 @@ +use super::matching_words::WordId; + +#[derive(Clone, Debug)] +pub enum MatchPosition { + Word { + // position of the word in the whole text. + word_position: usize, + // position of the token in the whole text. + token_position: usize, + }, + Phrase { + // position of the first and last word in the phrase in the whole text. + word_positions: [usize; 2], + // position of the first and last token in the phrase in the whole text. + token_positions: [usize; 2], + }, +} + +#[derive(Clone, Debug)] +pub struct Match { + pub char_count: usize, + // ids of the query words that matches. + pub ids: Vec, + pub position: MatchPosition, +} + +impl Match { + pub(super) fn get_first_word_pos(&self) -> usize { + match self.position { + MatchPosition::Word { word_position, .. } => word_position, + MatchPosition::Phrase { word_positions: [fwp, _], .. } => fwp, + } + } + + pub(super) fn get_last_word_pos(&self) -> usize { + match self.position { + MatchPosition::Word { word_position, .. } => word_position, + MatchPosition::Phrase { word_positions: [_, lwp], .. } => lwp, + } + } + + pub(super) fn get_first_token_pos(&self) -> usize { + match self.position { + MatchPosition::Word { token_position, .. } => token_position, + MatchPosition::Phrase { token_positions: [ftp, _], .. } => ftp, + } + } + + pub(super) fn get_last_token_pos(&self) -> usize { + match self.position { + MatchPosition::Word { token_position, .. } => token_position, + MatchPosition::Phrase { token_positions: [_, ltp], .. } => ltp, + } + } + + pub(super) fn get_word_count(&self) -> usize { + match self.position { + MatchPosition::Word { .. } => 1, + MatchPosition::Phrase { word_positions: [fwp, lwp], .. } => lwp - fwp + 1, + } + } +} diff --git a/milli/src/search/new/matches/matching_words.rs b/crates/milli/src/search/new/matches/matching_words.rs similarity index 91% rename from milli/src/search/new/matches/matching_words.rs rename to crates/milli/src/search/new/matches/matching_words.rs index 4ad5c37ec..1f30a17ad 100644 --- a/milli/src/search/new/matches/matching_words.rs +++ b/crates/milli/src/search/new/matches/matching_words.rs @@ -86,14 +86,17 @@ impl MatchingWords { continue; }; let prefix_length = char_index + c.len_utf8(); - let char_len = token.original_lengths(prefix_length).0; + let (char_count, byte_len) = token.original_lengths(prefix_length); let ids = &located_words.positions; - return Some(MatchType::Full { char_len, ids }); + return Some(MatchType::Full { ids, char_count, byte_len }); // else we exact match the token. } else if token.lemma() == word { - let char_len = token.char_end - token.char_start; let ids = &located_words.positions; - return Some(MatchType::Full { char_len, ids }); + return Some(MatchType::Full { + char_count: token.char_end - token.char_start, + byte_len: token.byte_end - token.byte_start, + ids, + }); } } } @@ -130,7 +133,7 @@ impl<'a> Iterator for MatchesIter<'a, '_> { word.map(|word| self.matching_words.word_interner.get(word).as_str()) }) .collect(); - let partial = PartialMatch { matching_words: words, ids, char_len: 0 }; + let partial = PartialMatch { matching_words: words, ids }; partial.match_token(self.token).or_else(|| self.next()) } @@ -149,7 +152,7 @@ pub type WordId = u16; /// In these cases we need to match consecutively several tokens to consider that the match is full. #[derive(Debug, PartialEq)] pub enum MatchType<'a> { - Full { char_len: usize, ids: &'a RangeInclusive }, + Full { char_count: usize, byte_len: usize, ids: &'a RangeInclusive }, Partial(PartialMatch<'a>), } @@ -158,7 +161,6 @@ pub enum MatchType<'a> { pub struct PartialMatch<'a> { matching_words: Vec>, ids: &'a RangeInclusive, - char_len: usize, } impl<'a> PartialMatch<'a> { @@ -176,25 +178,24 @@ impl<'a> PartialMatch<'a> { None => token.is_stopword(), }; - let char_len = token.char_end - token.char_start; // if there are remaining words to match in the phrase and the current token is matching, // return a new Partial match allowing the highlighter to continue. if is_matching && matching_words.len() > 1 { matching_words.remove(0); - Some(MatchType::Partial(PartialMatch { matching_words, ids, char_len })) + Some(MatchType::Partial(Self { matching_words, ids })) // if there is no remaining word to match in the phrase and the current token is matching, // return a Full match. } else if is_matching { - Some(MatchType::Full { char_len, ids }) + Some(MatchType::Full { + char_count: token.char_end - token.char_start, + byte_len: token.byte_end - token.byte_start, + ids, + }) // if the current token doesn't match, return None to break the match sequence. } else { None } } - - pub fn char_len(&self) -> usize { - self.char_len - } } impl fmt::Debug for MatchingWords { @@ -276,7 +277,7 @@ pub(crate) mod tests { ..Default::default() }) .next(), - Some(MatchType::Full { char_len: 5, ids: &(0..=0) }) + Some(MatchType::Full { char_count: 5, byte_len: 5, ids: &(0..=0) }) ); assert_eq!( matching_words @@ -300,7 +301,7 @@ pub(crate) mod tests { ..Default::default() }) .next(), - Some(MatchType::Full { char_len: 5, ids: &(2..=2) }) + Some(MatchType::Full { char_count: 5, byte_len: 5, ids: &(2..=2) }) ); assert_eq!( matching_words @@ -312,7 +313,7 @@ pub(crate) mod tests { ..Default::default() }) .next(), - Some(MatchType::Full { char_len: 5, ids: &(2..=2) }) + Some(MatchType::Full { char_count: 5, byte_len: 5, ids: &(2..=2) }) ); assert_eq!( matching_words diff --git a/milli/src/search/new/matches/mod.rs b/crates/milli/src/search/new/matches/mod.rs similarity index 74% rename from milli/src/search/new/matches/mod.rs rename to crates/milli/src/search/new/matches/mod.rs index 4688b8f32..80e3ec7b2 100644 --- a/milli/src/search/new/matches/mod.rs +++ b/crates/milli/src/search/new/matches/mod.rs @@ -1,11 +1,19 @@ -use std::borrow::Cow; +mod best_match_interval; +mod r#match; +mod matching_words; +mod simple_token_kind; use charabia::{Language, SeparatorKind, Token, Tokenizer}; +use either::Either; pub use matching_words::MatchingWords; -use matching_words::{MatchType, PartialMatch, WordId}; +use matching_words::{MatchType, PartialMatch}; +use r#match::{Match, MatchPosition}; use serde::Serialize; - -pub mod matching_words; +use simple_token_kind::SimpleTokenKind; +use std::{ + borrow::Cow, + cmp::{max, min}, +}; const DEFAULT_CROP_MARKER: &str = "…"; const DEFAULT_HIGHLIGHT_PREFIX: &str = ""; @@ -93,17 +101,6 @@ impl FormatOptions { } } -#[derive(Clone, Debug)] -pub struct Match { - match_len: usize, - // ids of the query words that matches. - ids: Vec, - // position of the word in the whole text. - word_position: usize, - // position of the token in the whole text. - token_position: usize, -} - #[derive(Serialize, Debug, Clone, PartialEq, Eq)] pub struct MatchBounds { pub start: usize, @@ -130,41 +127,27 @@ impl<'t, 'tokenizer> Matcher<'t, 'tokenizer, '_, '_> { /// compute_partial_match peek into next words to validate if the match is complete. fn compute_partial_match<'a>( mut partial: PartialMatch<'a>, - token_position: usize, - word_position: usize, + first_token_position: usize, + first_word_position: usize, + first_word_char_start: &usize, words_positions: &mut impl Iterator)>, matches: &mut Vec, ) -> bool { - let mut potential_matches = vec![(token_position, word_position, partial.char_len())]; - for (token_position, word_position, word) in words_positions { partial = match partial.match_token(word) { // token matches the partial match, but the match is not full, // we temporarily save the current token then we try to match the next one. - Some(MatchType::Partial(partial)) => { - potential_matches.push((token_position, word_position, partial.char_len())); - partial - } + Some(MatchType::Partial(partial)) => partial, // partial match is now full, we keep this matches and we advance positions - Some(MatchType::Full { char_len, ids }) => { - let ids: Vec<_> = ids.clone().collect(); - // save previously matched tokens as matches. - let iter = potential_matches.into_iter().map( - |(token_position, word_position, match_len)| Match { - match_len, - ids: ids.clone(), - word_position, - token_position, - }, - ); - matches.extend(iter); - + Some(MatchType::Full { ids, .. }) => { // save the token that closes the partial match as a match. matches.push(Match { - match_len: char_len, - ids, - word_position, - token_position, + char_count: word.char_end - *first_word_char_start, + ids: ids.clone().collect(), + position: MatchPosition::Phrase { + word_positions: [first_word_position, word_position], + token_positions: [first_token_position, token_position], + }, }); // the match is complete, we return true. @@ -202,13 +185,12 @@ impl<'t, 'tokenizer> Matcher<'t, 'tokenizer, '_, '_> { match match_type { // we match, we save the current token as a match, // then we continue the rest of the tokens. - MatchType::Full { char_len, ids } => { + MatchType::Full { ids, char_count, .. } => { let ids: Vec<_> = ids.clone().collect(); matches.push(Match { - match_len: char_len, + char_count, ids, - word_position, - token_position, + position: MatchPosition::Word { word_position, token_position }, }); break; } @@ -221,6 +203,7 @@ impl<'t, 'tokenizer> Matcher<'t, 'tokenizer, '_, '_> { partial, token_position, word_position, + &word.char_start, &mut wp, &mut matches, ) { @@ -243,56 +226,99 @@ impl<'t, 'tokenizer> Matcher<'t, 'tokenizer, '_, '_> { Some((tokens, matches)) => matches .iter() .map(|m| MatchBounds { - start: tokens[m.token_position].byte_start, - length: m.match_len, + start: tokens[m.get_first_token_pos()].byte_start, + // TODO: Why is this in chars, while start is in bytes? + length: m.char_count, }) .collect(), } } /// Returns the bounds in byte index of the crop window. - fn crop_bounds( - &self, - tokens: &[Token<'_>], - matches: &[Match], - crop_size: usize, - ) -> (usize, usize) { - // if there is no match, we start from the beginning of the string by default. - let first_match_word_position = matches.first().map(|m| m.word_position).unwrap_or(0); - let first_match_token_position = matches.first().map(|m| m.token_position).unwrap_or(0); - let last_match_word_position = matches.last().map(|m| m.word_position).unwrap_or(0); - let last_match_token_position = matches.last().map(|m| m.token_position).unwrap_or(0); + fn crop_bounds(&self, tokens: &[Token<'_>], matches: &[Match], crop_size: usize) -> [usize; 2] { + let ( + mut remaining_words, + is_iterating_forward, + before_tokens_starting_index, + after_tokens_starting_index, + ) = if !matches.is_empty() { + let [matches_first, matches_last] = + best_match_interval::find_best_match_interval(matches, crop_size); - // matches needs to be counted in the crop len. - let mut remaining_words = crop_size + first_match_word_position - last_match_word_position; + let matches_size = + matches_last.get_last_word_pos() - matches_first.get_first_word_pos() + 1; + + let is_crop_size_gte_match_size = crop_size >= matches_size; + let is_iterating_forward = matches_size == 0 || is_crop_size_gte_match_size; + + let remaining_words = if is_crop_size_gte_match_size { + crop_size - matches_size + } else { + // in case matches size is greater than crop size, which implies there's only one match, + // we count words backwards, because we have to remove words, as they're extra words outside of + // crop window + matches_size - crop_size + }; + + let after_tokens_starting_index = if matches_size == 0 { + 0 + } else { + let last_match_last_token_position_plus_one = matches_last.get_last_token_pos() + 1; + if last_match_last_token_position_plus_one < tokens.len() { + last_match_last_token_position_plus_one + } else { + // we have matched the end of possible tokens, there's nothing to advance + tokens.len() - 1 + } + }; + + ( + remaining_words, + is_iterating_forward, + if is_iterating_forward { matches_first.get_first_token_pos() } else { 0 }, + after_tokens_starting_index, + ) + } else { + (crop_size, true, 0, 0) + }; // create the initial state of the crop window: 2 iterators starting from the matches positions, // a reverse iterator starting from the first match token position and going towards the beginning of the text, - let mut before_tokens = tokens[..first_match_token_position].iter().rev().peekable(); - // an iterator starting from the last match token position and going towards the end of the text. - let mut after_tokens = tokens[last_match_token_position..].iter().peekable(); + let mut before_tokens = tokens[..before_tokens_starting_index].iter().rev().peekable(); + // an iterator ... + let mut after_tokens = if is_iterating_forward { + // ... starting from the last match token position and going towards the end of the text. + Either::Left(tokens[after_tokens_starting_index..].iter().peekable()) + } else { + // ... starting from the last match token position and going towards the start of the text. + Either::Right(tokens[..=after_tokens_starting_index].iter().rev().peekable()) + }; // grows the crop window peeking in both directions // until the window contains the good number of words: while remaining_words > 0 { - let before_token = before_tokens.peek().map(|t| t.separator_kind()); - let after_token = after_tokens.peek().map(|t| t.separator_kind()); + let before_token_kind = before_tokens.peek().map(SimpleTokenKind::new); + let after_token_kind = + after_tokens.as_mut().either(|v| v.peek(), |v| v.peek()).map(SimpleTokenKind::new); - match (before_token, after_token) { + match (before_token_kind, after_token_kind) { // we can expand both sides. - (Some(before_token), Some(after_token)) => { - match (before_token, after_token) { + (Some(before_token_kind), Some(after_token_kind)) => { + match (before_token_kind, after_token_kind) { // if they are both separators and are the same kind then advance both, // or expand in the soft separator separator side. - (Some(before_token_kind), Some(after_token_kind)) => { - if before_token_kind == after_token_kind { + ( + SimpleTokenKind::Separator(before_token_separator_kind), + SimpleTokenKind::Separator(after_token_separator_kind), + ) => { + if before_token_separator_kind == after_token_separator_kind { before_tokens.next(); // this avoid having an ending separator before crop marker. if remaining_words > 1 { after_tokens.next(); } - } else if before_token_kind == SeparatorKind::Hard { + } else if matches!(before_token_separator_kind, SeparatorKind::Hard) { after_tokens.next(); } else { before_tokens.next(); @@ -300,17 +326,17 @@ impl<'t, 'tokenizer> Matcher<'t, 'tokenizer, '_, '_> { } // if one of the tokens is a word, we expend in the side of the word. // left is a word, advance left. - (None, Some(_)) => { + (SimpleTokenKind::NotSeparator, SimpleTokenKind::Separator(_)) => { before_tokens.next(); remaining_words -= 1; } // right is a word, advance right. - (Some(_), None) => { + (SimpleTokenKind::Separator(_), SimpleTokenKind::NotSeparator) => { after_tokens.next(); remaining_words -= 1; } // both are words, advance left then right if remaining_word > 0. - (None, None) => { + (SimpleTokenKind::NotSeparator, SimpleTokenKind::NotSeparator) => { before_tokens.next(); remaining_words -= 1; @@ -322,16 +348,16 @@ impl<'t, 'tokenizer> Matcher<'t, 'tokenizer, '_, '_> { } } // the end of the text is reached, advance left. - (Some(before_token), None) => { + (Some(before_token_kind), None) => { before_tokens.next(); - if before_token.is_none() { + if matches!(before_token_kind, SimpleTokenKind::NotSeparator) { remaining_words -= 1; } } // the start of the text is reached, advance right. - (None, Some(after_token)) => { + (None, Some(after_token_kind)) => { after_tokens.next(); - if after_token.is_none() { + if matches!(after_token_kind, SimpleTokenKind::NotSeparator) { remaining_words -= 1; } } @@ -344,86 +370,7 @@ impl<'t, 'tokenizer> Matcher<'t, 'tokenizer, '_, '_> { let crop_byte_start = before_tokens.next().map_or(0, |t| t.byte_end); let crop_byte_end = after_tokens.next().map_or(self.text.len(), |t| t.byte_start); - (crop_byte_start, crop_byte_end) - } - - /// Compute the score of a match interval: - /// 1) count unique matches - /// 2) calculate distance between matches - /// 3) count ordered matches - fn match_interval_score(&self, matches: &[Match]) -> (i16, i16, i16) { - let mut ids: Vec = Vec::with_capacity(matches.len()); - let mut order_score = 0; - let mut distance_score = 0; - - let mut iter = matches.iter().peekable(); - while let Some(m) = iter.next() { - if let Some(next_match) = iter.peek() { - // if matches are ordered - if next_match.ids.iter().min() > m.ids.iter().min() { - order_score += 1; - } - - // compute distance between matches - distance_score -= (next_match.word_position - m.word_position).min(7) as i16; - } - - ids.extend(m.ids.iter()); - } - - ids.sort_unstable(); - ids.dedup(); - let uniq_score = ids.len() as i16; - - // rank by unique match count, then by distance between matches, then by ordered match count. - (uniq_score, distance_score, order_score) - } - - /// Returns the matches interval where the score computed by match_interval_score is the best. - fn find_best_match_interval<'a>(&self, matches: &'a [Match], crop_size: usize) -> &'a [Match] { - // we compute the matches interval if we have at least 2 matches. - if matches.len() > 1 { - // positions of the first and the last match of the best matches interval in `matches`. - let mut best_interval = (0, 0); - let mut best_interval_score = self.match_interval_score(&matches[0..=0]); - // current interval positions. - let mut interval_first = 0; - let mut interval_last = 0; - for (index, next_match) in matches.iter().enumerate().skip(1) { - // if next match would make interval gross more than crop_size, - // we compare the current interval with the best one, - // then we increase `interval_first` until next match can be added. - if next_match.word_position - matches[interval_first].word_position >= crop_size { - let interval_score = - self.match_interval_score(&matches[interval_first..=interval_last]); - - // keep interval if it's the best - if interval_score > best_interval_score { - best_interval = (interval_first, interval_last); - best_interval_score = interval_score; - } - - // advance start of the interval while interval is longer than crop_size. - while next_match.word_position - matches[interval_first].word_position - >= crop_size - { - interval_first += 1; - } - } - interval_last = index; - } - - // compute the last interval score and compare it to the best one. - let interval_score = - self.match_interval_score(&matches[interval_first..=interval_last]); - if interval_score > best_interval_score { - best_interval = (interval_first, interval_last); - } - - &matches[best_interval.0..=best_interval.1] - } else { - matches - } + [crop_byte_start, crop_byte_end] } // Returns the formatted version of the original text. @@ -434,69 +381,87 @@ impl<'t, 'tokenizer> Matcher<'t, 'tokenizer, '_, '_> { } else { match &self.matches { Some((tokens, matches)) => { - // If the text has to be cropped, - // crop around the best interval. - let (byte_start, byte_end) = match format_options.crop { + // If the text has to be cropped, crop around the best interval. + let [crop_byte_start, crop_byte_end] = match format_options.crop { Some(crop_size) if crop_size > 0 => { - let matches = self.find_best_match_interval(matches, crop_size); self.crop_bounds(tokens, matches, crop_size) } - _ => (0, self.text.len()), + _ => [0, self.text.len()], }; let mut formatted = Vec::new(); // push crop marker if it's not the start of the text. - if byte_start > 0 && !self.crop_marker.is_empty() { + if crop_byte_start > 0 && !self.crop_marker.is_empty() { formatted.push(self.crop_marker); } - let mut byte_index = byte_start; + let mut byte_index = crop_byte_start; if format_options.highlight { // insert highlight markers around matches. for m in matches { - let token = &tokens[m.token_position]; + let [m_byte_start, m_byte_end] = match m.position { + MatchPosition::Word { token_position, .. } => { + let token = &tokens[token_position]; + [&token.byte_start, &token.byte_end] + } + MatchPosition::Phrase { token_positions: [ftp, ltp], .. } => { + [&tokens[ftp].byte_start, &tokens[ltp].byte_end] + } + }; - // skip matches out of the crop window. - if token.byte_start < byte_start || token.byte_end > byte_end { + // skip matches out of the crop window + if *m_byte_end < crop_byte_start || *m_byte_start > crop_byte_end { continue; } - if byte_index < token.byte_start { - formatted.push(&self.text[byte_index..token.byte_start]); + // adjust start and end to the crop window size + let [m_byte_start, m_byte_end] = [ + max(m_byte_start, &crop_byte_start), + min(m_byte_end, &crop_byte_end), + ]; + + // push text that is positioned before our matches + if byte_index < *m_byte_start { + formatted.push(&self.text[byte_index..*m_byte_start]); } - let highlight_byte_index = self.text[token.byte_start..] - .char_indices() - .enumerate() - .find(|(i, _)| *i == m.match_len) - .map_or(token.byte_end, |(_, (i, _))| i + token.byte_start); formatted.push(self.highlight_prefix); - formatted.push(&self.text[token.byte_start..highlight_byte_index]); + + // TODO: This is additional work done, charabia::token::Token byte_len + // should already get us the original byte length, however, that doesn't work as + // it's supposed to, investigate why + let highlight_byte_index = self.text[*m_byte_start..] + .char_indices() + .nth(m.char_count) + .map_or(*m_byte_end, |(i, _)| min(i + *m_byte_start, *m_byte_end)); + formatted.push(&self.text[*m_byte_start..highlight_byte_index]); + formatted.push(self.highlight_suffix); + // if it's a prefix highlight, we put the end of the word after the highlight marker. - if highlight_byte_index < token.byte_end { - formatted.push(&self.text[highlight_byte_index..token.byte_end]); + if highlight_byte_index < *m_byte_end { + formatted.push(&self.text[highlight_byte_index..*m_byte_end]); } - byte_index = token.byte_end; + byte_index = *m_byte_end; } } // push the rest of the text between last match and the end of crop. - if byte_index < byte_end { - formatted.push(&self.text[byte_index..byte_end]); + if byte_index < crop_byte_end { + formatted.push(&self.text[byte_index..crop_byte_end]); } // push crop marker if it's not the end of the text. - if byte_end < self.text.len() && !self.crop_marker.is_empty() { + if crop_byte_end < self.text.len() && !self.crop_marker.is_empty() { formatted.push(self.crop_marker); } if formatted.len() == 1 { // avoid concatenating if there is already 1 slice. - Cow::Borrowed(&self.text[byte_start..byte_end]) + Cow::Borrowed(&self.text[crop_byte_start..crop_byte_end]) } else { Cow::Owned(formatted.concat()) } @@ -821,22 +786,24 @@ mod tests { fn format_highlight_crop_phrase_query() { //! testing: https://github.com/meilisearch/meilisearch/issues/3975 let temp_index = TempIndex::new(); + + let text = "The groundbreaking invention had the power to split the world between those who embraced progress and those who resisted change!"; temp_index .add_documents(documents!([ - { "id": 1, "text": "The groundbreaking invention had the power to split the world between those who embraced progress and those who resisted change!" } + { "id": 1, "text": text } ])) .unwrap(); + let rtxn = temp_index.read_txn().unwrap(); let format_options = FormatOptions { highlight: true, crop: Some(10) }; - let text = "The groundbreaking invention had the power to split the world between those who embraced progress and those who resisted change!"; let builder = MatcherBuilder::new_test(&rtxn, &temp_index, "\"the world\""); let mut matcher = builder.build(text, None); // should return 10 words with a marker at the start as well the end, and the highlighted matches. insta::assert_snapshot!( matcher.format(format_options), - @"…had the power to split the world between those who…" + @"…the power to split the world between those who embraced…" ); let builder = MatcherBuilder::new_test(&rtxn, &temp_index, "those \"and those\""); @@ -844,7 +811,63 @@ mod tests { // should highlight "those" and the phrase "and those". insta::assert_snapshot!( matcher.format(format_options), - @"…world between those who embraced progress and those who resisted…" + @"…world between those who embraced progress and those who resisted…" + ); + + let builder = MatcherBuilder::new_test( + &rtxn, + &temp_index, + "\"The groundbreaking invention had the power to split the world\"", + ); + let mut matcher = builder.build(text, None); + insta::assert_snapshot!( + matcher.format(format_options), + @"The groundbreaking invention had the power to split the world…" + ); + + let builder = MatcherBuilder::new_test( + &rtxn, + &temp_index, + "\"The groundbreaking invention had the power to split the world between those\"", + ); + let mut matcher = builder.build(text, None); + insta::assert_snapshot!( + matcher.format(format_options), + @"The groundbreaking invention had the power to split the world…" + ); + + let builder = MatcherBuilder::new_test( + &rtxn, + &temp_index, + "\"The groundbreaking invention\" \"embraced progress and those who resisted change!\"", + ); + let mut matcher = builder.build(text, None); + insta::assert_snapshot!( + matcher.format(format_options), + // TODO: Should include exclamation mark without crop markers + @"…between those who embraced progress and those who resisted change…" + ); + + let builder = MatcherBuilder::new_test( + &rtxn, + &temp_index, + "\"groundbreaking invention\" \"split the world between\"", + ); + let mut matcher = builder.build(text, None); + insta::assert_snapshot!( + matcher.format(format_options), + @"…groundbreaking invention had the power to split the world between…" + ); + + let builder = MatcherBuilder::new_test( + &rtxn, + &temp_index, + "\"groundbreaking invention\" \"had the power to split the world between those\"", + ); + let mut matcher = builder.build(text, None); + insta::assert_snapshot!( + matcher.format(format_options), + @"…invention had the power to split the world between those…" ); } @@ -900,7 +923,7 @@ mod tests { let mut matcher = builder.build(text, None); insta::assert_snapshot!( matcher.format(format_options), - @"_the_ _do_ _or_ die can't be he do and or isn'_t_ _he_" + @"_the_ _do or_ die can't be he do and or isn'_t he_" ); } } diff --git a/crates/milli/src/search/new/matches/simple_token_kind.rs b/crates/milli/src/search/new/matches/simple_token_kind.rs new file mode 100644 index 000000000..b34a8c985 --- /dev/null +++ b/crates/milli/src/search/new/matches/simple_token_kind.rs @@ -0,0 +1,15 @@ +use charabia::{SeparatorKind, Token, TokenKind}; + +pub enum SimpleTokenKind { + Separator(SeparatorKind), + NotSeparator, +} + +impl SimpleTokenKind { + pub fn new(token: &&Token<'_>) -> Self { + match token.kind { + TokenKind::Separator(separaor_kind) => Self::Separator(separaor_kind), + _ => Self::NotSeparator, + } + } +} diff --git a/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs similarity index 100% rename from milli/src/search/new/mod.rs rename to crates/milli/src/search/new/mod.rs diff --git a/milli/src/search/new/query_graph.rs b/crates/milli/src/search/new/query_graph.rs similarity index 100% rename from milli/src/search/new/query_graph.rs rename to crates/milli/src/search/new/query_graph.rs diff --git a/milli/src/search/new/query_term/compute_derivations.rs b/crates/milli/src/search/new/query_term/compute_derivations.rs similarity index 100% rename from milli/src/search/new/query_term/compute_derivations.rs rename to crates/milli/src/search/new/query_term/compute_derivations.rs diff --git a/milli/src/search/new/query_term/mod.rs b/crates/milli/src/search/new/query_term/mod.rs similarity index 100% rename from milli/src/search/new/query_term/mod.rs rename to crates/milli/src/search/new/query_term/mod.rs diff --git a/milli/src/search/new/query_term/ntypo_subset.rs b/crates/milli/src/search/new/query_term/ntypo_subset.rs similarity index 100% rename from milli/src/search/new/query_term/ntypo_subset.rs rename to crates/milli/src/search/new/query_term/ntypo_subset.rs diff --git a/milli/src/search/new/query_term/parse_query.rs b/crates/milli/src/search/new/query_term/parse_query.rs similarity index 100% rename from milli/src/search/new/query_term/parse_query.rs rename to crates/milli/src/search/new/query_term/parse_query.rs diff --git a/milli/src/search/new/query_term/phrase.rs b/crates/milli/src/search/new/query_term/phrase.rs similarity index 100% rename from milli/src/search/new/query_term/phrase.rs rename to crates/milli/src/search/new/query_term/phrase.rs diff --git a/milli/src/search/new/ranking_rule_graph/build.rs b/crates/milli/src/search/new/ranking_rule_graph/build.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/build.rs rename to crates/milli/src/search/new/ranking_rule_graph/build.rs diff --git a/milli/src/search/new/ranking_rule_graph/cheapest_paths.rs b/crates/milli/src/search/new/ranking_rule_graph/cheapest_paths.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/cheapest_paths.rs rename to crates/milli/src/search/new/ranking_rule_graph/cheapest_paths.rs diff --git a/milli/src/search/new/ranking_rule_graph/condition_docids_cache.rs b/crates/milli/src/search/new/ranking_rule_graph/condition_docids_cache.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/condition_docids_cache.rs rename to crates/milli/src/search/new/ranking_rule_graph/condition_docids_cache.rs diff --git a/milli/src/search/new/ranking_rule_graph/dead_ends_cache.rs b/crates/milli/src/search/new/ranking_rule_graph/dead_ends_cache.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/dead_ends_cache.rs rename to crates/milli/src/search/new/ranking_rule_graph/dead_ends_cache.rs diff --git a/milli/src/search/new/ranking_rule_graph/exactness/mod.rs b/crates/milli/src/search/new/ranking_rule_graph/exactness/mod.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/exactness/mod.rs rename to crates/milli/src/search/new/ranking_rule_graph/exactness/mod.rs diff --git a/milli/src/search/new/ranking_rule_graph/fid/mod.rs b/crates/milli/src/search/new/ranking_rule_graph/fid/mod.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/fid/mod.rs rename to crates/milli/src/search/new/ranking_rule_graph/fid/mod.rs diff --git a/milli/src/search/new/ranking_rule_graph/mod.rs b/crates/milli/src/search/new/ranking_rule_graph/mod.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/mod.rs rename to crates/milli/src/search/new/ranking_rule_graph/mod.rs diff --git a/milli/src/search/new/ranking_rule_graph/position/mod.rs b/crates/milli/src/search/new/ranking_rule_graph/position/mod.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/position/mod.rs rename to crates/milli/src/search/new/ranking_rule_graph/position/mod.rs diff --git a/milli/src/search/new/ranking_rule_graph/proximity/build.rs b/crates/milli/src/search/new/ranking_rule_graph/proximity/build.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/proximity/build.rs rename to crates/milli/src/search/new/ranking_rule_graph/proximity/build.rs diff --git a/milli/src/search/new/ranking_rule_graph/proximity/compute_docids.rs b/crates/milli/src/search/new/ranking_rule_graph/proximity/compute_docids.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/proximity/compute_docids.rs rename to crates/milli/src/search/new/ranking_rule_graph/proximity/compute_docids.rs diff --git a/milli/src/search/new/ranking_rule_graph/proximity/mod.rs b/crates/milli/src/search/new/ranking_rule_graph/proximity/mod.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/proximity/mod.rs rename to crates/milli/src/search/new/ranking_rule_graph/proximity/mod.rs diff --git a/milli/src/search/new/ranking_rule_graph/typo/mod.rs b/crates/milli/src/search/new/ranking_rule_graph/typo/mod.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/typo/mod.rs rename to crates/milli/src/search/new/ranking_rule_graph/typo/mod.rs diff --git a/milli/src/search/new/ranking_rule_graph/words/mod.rs b/crates/milli/src/search/new/ranking_rule_graph/words/mod.rs similarity index 100% rename from milli/src/search/new/ranking_rule_graph/words/mod.rs rename to crates/milli/src/search/new/ranking_rule_graph/words/mod.rs diff --git a/milli/src/search/new/ranking_rules.rs b/crates/milli/src/search/new/ranking_rules.rs similarity index 100% rename from milli/src/search/new/ranking_rules.rs rename to crates/milli/src/search/new/ranking_rules.rs diff --git a/milli/src/search/new/resolve_query_graph.rs b/crates/milli/src/search/new/resolve_query_graph.rs similarity index 100% rename from milli/src/search/new/resolve_query_graph.rs rename to crates/milli/src/search/new/resolve_query_graph.rs diff --git a/milli/src/search/new/small_bitmap.rs b/crates/milli/src/search/new/small_bitmap.rs similarity index 100% rename from milli/src/search/new/small_bitmap.rs rename to crates/milli/src/search/new/small_bitmap.rs diff --git a/milli/src/search/new/sort.rs b/crates/milli/src/search/new/sort.rs similarity index 100% rename from milli/src/search/new/sort.rs rename to crates/milli/src/search/new/sort.rs diff --git a/milli/src/search/new/tests/attribute_fid.rs b/crates/milli/src/search/new/tests/attribute_fid.rs similarity index 100% rename from milli/src/search/new/tests/attribute_fid.rs rename to crates/milli/src/search/new/tests/attribute_fid.rs diff --git a/milli/src/search/new/tests/attribute_position.rs b/crates/milli/src/search/new/tests/attribute_position.rs similarity index 100% rename from milli/src/search/new/tests/attribute_position.rs rename to crates/milli/src/search/new/tests/attribute_position.rs diff --git a/milli/src/search/new/tests/cutoff.rs b/crates/milli/src/search/new/tests/cutoff.rs similarity index 100% rename from milli/src/search/new/tests/cutoff.rs rename to crates/milli/src/search/new/tests/cutoff.rs diff --git a/milli/src/search/new/tests/distinct.rs b/crates/milli/src/search/new/tests/distinct.rs similarity index 100% rename from milli/src/search/new/tests/distinct.rs rename to crates/milli/src/search/new/tests/distinct.rs diff --git a/milli/src/search/new/tests/exactness.rs b/crates/milli/src/search/new/tests/exactness.rs similarity index 100% rename from milli/src/search/new/tests/exactness.rs rename to crates/milli/src/search/new/tests/exactness.rs diff --git a/milli/src/search/new/tests/geo_sort.rs b/crates/milli/src/search/new/tests/geo_sort.rs similarity index 100% rename from milli/src/search/new/tests/geo_sort.rs rename to crates/milli/src/search/new/tests/geo_sort.rs diff --git a/milli/src/search/new/tests/integration.rs b/crates/milli/src/search/new/tests/integration.rs similarity index 100% rename from milli/src/search/new/tests/integration.rs rename to crates/milli/src/search/new/tests/integration.rs diff --git a/milli/src/search/new/tests/language.rs b/crates/milli/src/search/new/tests/language.rs similarity index 100% rename from milli/src/search/new/tests/language.rs rename to crates/milli/src/search/new/tests/language.rs diff --git a/milli/src/search/new/tests/mod.rs b/crates/milli/src/search/new/tests/mod.rs similarity index 100% rename from milli/src/search/new/tests/mod.rs rename to crates/milli/src/search/new/tests/mod.rs diff --git a/milli/src/search/new/tests/ngram_split_words.rs b/crates/milli/src/search/new/tests/ngram_split_words.rs similarity index 100% rename from milli/src/search/new/tests/ngram_split_words.rs rename to crates/milli/src/search/new/tests/ngram_split_words.rs diff --git a/milli/src/search/new/tests/proximity.rs b/crates/milli/src/search/new/tests/proximity.rs similarity index 100% rename from milli/src/search/new/tests/proximity.rs rename to crates/milli/src/search/new/tests/proximity.rs diff --git a/milli/src/search/new/tests/proximity_typo.rs b/crates/milli/src/search/new/tests/proximity_typo.rs similarity index 100% rename from milli/src/search/new/tests/proximity_typo.rs rename to crates/milli/src/search/new/tests/proximity_typo.rs diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_fid__attribute_fid_ngrams-4.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_fid__attribute_fid_ngrams-4.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_fid__attribute_fid_ngrams-4.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_fid__attribute_fid_ngrams-4.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_fid__attribute_fid_simple.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_fid__attribute_fid_simple.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_fid__attribute_fid_simple.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_fid__attribute_fid_simple.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_different_fields.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_different_fields.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_different_fields.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_different_fields.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_ngrams.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_ngrams.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_ngrams.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_ngrams.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_repeated.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_repeated.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_repeated.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_repeated.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_simple-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_simple-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_simple-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__attribute_position__attribute_position_simple-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort-4.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort-4.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort-4.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort-4.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-10.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-10.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-10.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-10.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-12.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-12.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-12.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-12.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-14.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-14.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-14.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-14.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-16.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-16.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-16.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-16.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-18.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-18.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-18.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-18.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-20.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-20.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-20.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-20.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-4.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-4.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-4.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-4.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-6.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-6.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-6.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-6.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-8.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-8.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-8.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_around_the_edge_of_the_flat_earth-8.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_mixed_with_words-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_mixed_with_words-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_mixed_with_words-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_mixed_with_words-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_without_any_geo_faceted_documents-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_without_any_geo_faceted_documents-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_without_any_geo_faceted_documents-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__geo_sort__geo_sort_without_any_geo_faceted_documents-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-11.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-11.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-11.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-11.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-14.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-14.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-14.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-14.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-5.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-5.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-5.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-5.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-8.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-8.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-8.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_prefix_db-8.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-5.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-5.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-5.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-5.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-8.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-8.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-8.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__proximity__proximity_split_word-8.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__redacted-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__redacted-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__redacted-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__redacted-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-11.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-11.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-11.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-11.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-5.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-5.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-5.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-5.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-8.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-8.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-8.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__sort__sort-8.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-8.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-8.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-8.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-8.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-5.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-5.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-5.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-5.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-8.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-8.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-8.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_bucketing-8.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_exact_attribute-4.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_exact_attribute-4.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_exact_attribute-4.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_exact_attribute-4.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_exact_word-12.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_exact_word-12.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_exact_word-12.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_exact_word-12.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_ranking_rule_not_preceded_by_words_ranking_rule-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_ranking_rule_not_preceded_by_words_ranking_rule-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_ranking_rule_not_preceded_by_words_ranking_rule-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_ranking_rule_not_preceded_by_words_ranking_rule-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_ranking_rule_not_preceded_by_words_ranking_rule-5.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_ranking_rule_not_preceded_by_words_ranking_rule-5.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_ranking_rule_not_preceded_by_words_ranking_rule-5.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_ranking_rule_not_preceded_by_words_ranking_rule-5.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_synonyms-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_synonyms-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_synonyms-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_synonyms-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_synonyms-5.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_synonyms-5.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_synonyms-5.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo__typo_synonyms-5.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo_proximity__trap_basic_and_complex1-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo_proximity__trap_basic_and_complex1-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo_proximity__trap_basic_and_complex1-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo_proximity__trap_basic_and_complex1-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo_proximity__trap_complex2-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo_proximity__trap_complex2-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__typo_proximity__trap_complex2-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__typo_proximity__trap_complex2-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_phrase-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_phrase-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_phrase-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_phrase-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_phrase-5.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_phrase-5.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_phrase-5.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_phrase-5.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_simple-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_simple-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_simple-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_simple-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_simple-5.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_simple-5.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_simple-5.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_proximity_tms_last_simple-5.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_all-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_all-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_all-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_all-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_phrase-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_phrase-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_phrase-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_phrase-2.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_phrase-5.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_phrase-5.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_phrase-5.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_phrase-5.snap diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_simple-2.snap b/crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_simple-2.snap similarity index 100% rename from milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_simple-2.snap rename to crates/milli/src/search/new/tests/snapshots/milli__search__new__tests__words_tms__words_tms_last_simple-2.snap diff --git a/milli/src/search/new/tests/sort.rs b/crates/milli/src/search/new/tests/sort.rs similarity index 100% rename from milli/src/search/new/tests/sort.rs rename to crates/milli/src/search/new/tests/sort.rs diff --git a/milli/src/search/new/tests/stop_words.rs b/crates/milli/src/search/new/tests/stop_words.rs similarity index 100% rename from milli/src/search/new/tests/stop_words.rs rename to crates/milli/src/search/new/tests/stop_words.rs diff --git a/milli/src/search/new/tests/typo.rs b/crates/milli/src/search/new/tests/typo.rs similarity index 100% rename from milli/src/search/new/tests/typo.rs rename to crates/milli/src/search/new/tests/typo.rs diff --git a/milli/src/search/new/tests/typo_proximity.rs b/crates/milli/src/search/new/tests/typo_proximity.rs similarity index 100% rename from milli/src/search/new/tests/typo_proximity.rs rename to crates/milli/src/search/new/tests/typo_proximity.rs diff --git a/milli/src/search/new/tests/words_tms.rs b/crates/milli/src/search/new/tests/words_tms.rs similarity index 100% rename from milli/src/search/new/tests/words_tms.rs rename to crates/milli/src/search/new/tests/words_tms.rs diff --git a/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs similarity index 90% rename from milli/src/search/new/vector_sort.rs rename to crates/milli/src/search/new/vector_sort.rs index de1dacbe7..90377c09c 100644 --- a/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -1,11 +1,10 @@ use std::iter::FromIterator; -use ordered_float::OrderedFloat; use roaring::RoaringBitmap; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; use crate::score_details::{self, ScoreDetails}; -use crate::vector::{DistributionShift, Embedder}; +use crate::vector::{ArroyWrapper, DistributionShift, Embedder}; use crate::{DocumentId, Result, SearchContext, SearchLogger}; pub struct VectorSort { @@ -53,14 +52,9 @@ impl VectorSort { vector_candidates: &RoaringBitmap, ) -> Result<()> { let target = &self.target; - let mut results = Vec::new(); - for reader in ctx.index.arroy_readers(ctx.txn, self.embedder_index, self.quantized) { - let nns_by_vector = - reader?.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?; - results.extend(nns_by_vector.into_iter()); - } - results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + let reader = ArroyWrapper::new(ctx.index.vector_arroy, self.embedder_index, self.quantized); + let results = reader.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?; self.cached_sorted_docids = results.into_iter(); Ok(()) diff --git a/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs similarity index 86% rename from milli/src/search/similar.rs rename to crates/milli/src/search/similar.rs index 0cb8d723d..5547d800e 100644 --- a/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -1,10 +1,9 @@ use std::sync::Arc; -use ordered_float::OrderedFloat; use roaring::RoaringBitmap; use crate::score_details::{self, ScoreDetails}; -use crate::vector::Embedder; +use crate::vector::{ArroyWrapper, Embedder}; use crate::{filtered_universe, DocumentId, Filter, Index, Result, SearchResult}; pub struct Similar<'a> { @@ -71,23 +70,13 @@ impl<'a> Similar<'a> { .get(self.rtxn, &self.embedder_name)? .ok_or_else(|| crate::UserError::InvalidEmbedder(self.embedder_name.to_owned()))?; - let mut results = Vec::new(); - - for reader in self.index.arroy_readers(self.rtxn, embedder_index, self.quantized) { - let nns_by_item = reader?.nns_by_item( - self.rtxn, - self.id, - self.limit + self.offset + 1, - Some(&universe), - )?; - if let Some(mut nns_by_item) = nns_by_item { - results.append(&mut nns_by_item); - } else { - break; - } - } - - results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + let reader = ArroyWrapper::new(self.index.vector_arroy, embedder_index, self.quantized); + let results = reader.nns_by_item( + self.rtxn, + self.id, + self.limit + self.offset + 1, + Some(&universe), + )?; let mut documents_ids = Vec::with_capacity(self.limit); let mut document_scores = Vec::with_capacity(self.limit); diff --git a/milli/src/snapshot_tests.rs b/crates/milli/src/snapshot_tests.rs similarity index 100% rename from milli/src/snapshot_tests.rs rename to crates/milli/src/snapshot_tests.rs diff --git a/milli/src/snapshots/index.rs/bug_3007/geo_faceted_documents_ids.snap b/crates/milli/src/snapshots/index.rs/bug_3007/geo_faceted_documents_ids.snap similarity index 100% rename from milli/src/snapshots/index.rs/bug_3007/geo_faceted_documents_ids.snap rename to crates/milli/src/snapshots/index.rs/bug_3007/geo_faceted_documents_ids.snap diff --git a/milli/src/snapshots/index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap b/crates/milli/src/snapshots/index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap similarity index 100% rename from milli/src/snapshots/index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap rename to crates/milli/src/snapshots/index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap diff --git a/milli/src/thread_pool_no_abort.rs b/crates/milli/src/thread_pool_no_abort.rs similarity index 100% rename from milli/src/thread_pool_no_abort.rs rename to crates/milli/src/thread_pool_no_abort.rs diff --git a/crates/milli/src/update/available_documents_ids.rs b/crates/milli/src/update/available_documents_ids.rs new file mode 100644 index 000000000..e69de29bb diff --git a/milli/src/update/available_ids.rs b/crates/milli/src/update/available_ids.rs similarity index 100% rename from milli/src/update/available_ids.rs rename to crates/milli/src/update/available_ids.rs diff --git a/milli/src/update/clear_documents.rs b/crates/milli/src/update/clear_documents.rs similarity index 100% rename from milli/src/update/clear_documents.rs rename to crates/milli/src/update/clear_documents.rs diff --git a/milli/src/update/concurrent_available_ids.rs b/crates/milli/src/update/concurrent_available_ids.rs similarity index 100% rename from milli/src/update/concurrent_available_ids.rs rename to crates/milli/src/update/concurrent_available_ids.rs diff --git a/milli/src/update/del_add.rs b/crates/milli/src/update/del_add.rs similarity index 100% rename from milli/src/update/del_add.rs rename to crates/milli/src/update/del_add.rs diff --git a/milli/src/update/facet/bulk.rs b/crates/milli/src/update/facet/bulk.rs similarity index 100% rename from milli/src/update/facet/bulk.rs rename to crates/milli/src/update/facet/bulk.rs diff --git a/milli/src/update/facet/incremental.rs b/crates/milli/src/update/facet/incremental.rs similarity index 100% rename from milli/src/update/facet/incremental.rs rename to crates/milli/src/update/facet/incremental.rs diff --git a/milli/src/update/facet/mod.rs b/crates/milli/src/update/facet/mod.rs similarity index 100% rename from milli/src/update/facet/mod.rs rename to crates/milli/src/update/facet/mod.rs diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert/default.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert/default.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert/default.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert/default.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert/large_group_small_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert/large_group_small_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert/large_group_small_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert/large_group_small_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert/odd_group_odd_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert/odd_group_odd_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert/odd_group_odd_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert/odd_group_odd_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert/small_group_large_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert/small_group_large_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert/small_group_large_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert/small_group_large_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert/small_group_small_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert/small_group_small_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert/small_group_small_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert/small_group_small_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/default.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/default.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/default.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/default.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/large_group_small_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/large_group_small_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/large_group_small_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/large_group_small_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/odd_group_odd_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/odd_group_odd_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/odd_group_odd_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/odd_group_odd_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/small_group_large_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/small_group_large_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/small_group_large_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/small_group_large_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/small_group_small_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/small_group_small_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/small_group_small_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_delete_field_insert/small_group_small_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_string/default.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/default.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_string/default.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/default.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_string/large_group_small_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/large_group_small_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_string/large_group_small_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/large_group_small_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_string/odd_group_odd_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/odd_group_odd_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_string/odd_group_odd_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/odd_group_odd_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_string/small_group_large_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/small_group_large_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_string/small_group_large_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/small_group_large_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/bulk.rs/insert_string/small_group_small_min_level.hash.snap b/crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/small_group_small_min_level.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/bulk.rs/insert_string/small_group_small_min_level.hash.snap rename to crates/milli/src/update/facet/snapshots/bulk.rs/insert_string/small_group_small_min_level.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/append/append.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/append/append.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/append/append.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/append/append.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/0.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/0.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_from_end/0.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/0.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/100.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/100.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_from_end/100.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/100.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/15.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/15.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_from_end/15.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/15.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/150.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/150.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_from_end/150.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/150.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/17.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/17.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_from_end/17.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/17.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/200.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/200.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_from_end/200.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_end/200.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/127.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/127.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_from_start/127.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/127.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/215.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/215.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_from_start/215.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/215.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/255.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/255.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_from_start/255.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_from_start/255.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/127.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/127.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/127.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/127.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/215.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/215.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/215.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/215.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/255.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/255.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/255.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/delete_shuffled/255.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/after_delete.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/after_delete.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/after_delete.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/after_delete.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/before_delete.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/before_delete.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/before_delete.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_delete/before_delete.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_insert/in_place_level0_insert.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_insert/in_place_level0_insert.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/in_place_level0_insert/in_place_level0_insert.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/in_place_level0_insert/in_place_level0_insert.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_append/many_field_ids_append.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_append/many_field_ids_append.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/many_field_ids_append/many_field_ids_append.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_append/many_field_ids_append.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_prepend/many_field_ids_prepend.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_prepend/many_field_ids_prepend.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/many_field_ids_prepend/many_field_ids_prepend.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/many_field_ids_prepend/many_field_ids_prepend.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/merge_values/merge_values.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/merge_values/merge_values.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/merge_values/merge_values.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/merge_values/merge_values.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/prepend/prepend.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/prepend/prepend.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/prepend/prepend.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/prepend/prepend.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string_and_delete/after_delete.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string_and_delete/after_delete.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string_and_delete/after_delete.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string_and_delete/after_delete.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string_and_delete/before_delete.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string_and_delete/before_delete.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string_and_delete/before_delete.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/shuffle_merge_string_and_delete/before_delete.hash.snap diff --git a/milli/src/update/facet/snapshots/incremental.rs/shuffled/shuffled.hash.snap b/crates/milli/src/update/facet/snapshots/incremental.rs/shuffled/shuffled.hash.snap similarity index 100% rename from milli/src/update/facet/snapshots/incremental.rs/shuffled/shuffled.hash.snap rename to crates/milli/src/update/facet/snapshots/incremental.rs/shuffled/shuffled.hash.snap diff --git a/milli/src/update/index_documents/enrich.rs b/crates/milli/src/update/index_documents/enrich.rs similarity index 100% rename from milli/src/update/index_documents/enrich.rs rename to crates/milli/src/update/index_documents/enrich.rs diff --git a/milli/src/update/index_documents/extract/extract_docid_word_positions.rs b/crates/milli/src/update/index_documents/extract/extract_docid_word_positions.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_docid_word_positions.rs rename to crates/milli/src/update/index_documents/extract/extract_docid_word_positions.rs diff --git a/milli/src/update/index_documents/extract/extract_facet_number_docids.rs b/crates/milli/src/update/index_documents/extract/extract_facet_number_docids.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_facet_number_docids.rs rename to crates/milli/src/update/index_documents/extract/extract_facet_number_docids.rs diff --git a/milli/src/update/index_documents/extract/extract_facet_string_docids.rs b/crates/milli/src/update/index_documents/extract/extract_facet_string_docids.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_facet_string_docids.rs rename to crates/milli/src/update/index_documents/extract/extract_facet_string_docids.rs diff --git a/milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs b/crates/milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs rename to crates/milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs diff --git a/milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs b/crates/milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs rename to crates/milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs diff --git a/milli/src/update/index_documents/extract/extract_geo_points.rs b/crates/milli/src/update/index_documents/extract/extract_geo_points.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_geo_points.rs rename to crates/milli/src/update/index_documents/extract/extract_geo_points.rs diff --git a/milli/src/update/index_documents/extract/extract_vector_points.rs b/crates/milli/src/update/index_documents/extract/extract_vector_points.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_vector_points.rs rename to crates/milli/src/update/index_documents/extract/extract_vector_points.rs diff --git a/milli/src/update/index_documents/extract/extract_word_docids.rs b/crates/milli/src/update/index_documents/extract/extract_word_docids.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_word_docids.rs rename to crates/milli/src/update/index_documents/extract/extract_word_docids.rs diff --git a/milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs b/crates/milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs rename to crates/milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs diff --git a/milli/src/update/index_documents/extract/extract_word_position_docids.rs b/crates/milli/src/update/index_documents/extract/extract_word_position_docids.rs similarity index 100% rename from milli/src/update/index_documents/extract/extract_word_position_docids.rs rename to crates/milli/src/update/index_documents/extract/extract_word_position_docids.rs diff --git a/milli/src/update/index_documents/extract/mod.rs b/crates/milli/src/update/index_documents/extract/mod.rs similarity index 100% rename from milli/src/update/index_documents/extract/mod.rs rename to crates/milli/src/update/index_documents/extract/mod.rs diff --git a/milli/src/update/index_documents/helpers/clonable_mmap.rs b/crates/milli/src/update/index_documents/helpers/clonable_mmap.rs similarity index 100% rename from milli/src/update/index_documents/helpers/clonable_mmap.rs rename to crates/milli/src/update/index_documents/helpers/clonable_mmap.rs diff --git a/milli/src/update/index_documents/helpers/grenad_helpers.rs b/crates/milli/src/update/index_documents/helpers/grenad_helpers.rs similarity index 100% rename from milli/src/update/index_documents/helpers/grenad_helpers.rs rename to crates/milli/src/update/index_documents/helpers/grenad_helpers.rs diff --git a/milli/src/update/index_documents/helpers/merge_functions.rs b/crates/milli/src/update/index_documents/helpers/merge_functions.rs similarity index 100% rename from milli/src/update/index_documents/helpers/merge_functions.rs rename to crates/milli/src/update/index_documents/helpers/merge_functions.rs diff --git a/milli/src/update/index_documents/helpers/mod.rs b/crates/milli/src/update/index_documents/helpers/mod.rs similarity index 100% rename from milli/src/update/index_documents/helpers/mod.rs rename to crates/milli/src/update/index_documents/helpers/mod.rs diff --git a/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs similarity index 99% rename from milli/src/update/index_documents/mod.rs rename to crates/milli/src/update/index_documents/mod.rs index 49afe9ac5..befde896d 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -683,9 +683,8 @@ where key: None, }, )?; - let first_id = crate::vector::arroy_db_range_for_embedder(index).next().unwrap(); let reader = - ArroyWrapper::new(self.index.vector_arroy, first_id, action.was_quantized); + ArroyWrapper::new(self.index.vector_arroy, index, action.was_quantized); let dim = reader.dimensions(self.wtxn)?; dimension.insert(name.to_string(), dim); } @@ -694,6 +693,7 @@ where for (embedder_name, dimension) in dimension { let wtxn = &mut *self.wtxn; let vector_arroy = self.index.vector_arroy; + let cancel = &self.should_abort; let embedder_index = self.index.embedder_category_id.get(wtxn, &embedder_name)?.ok_or( InternalError::DatabaseMissingEntry { db_name: "embedder_category_id", key: None }, @@ -707,17 +707,8 @@ where let is_quantizing = embedder_config.map_or(false, |action| action.is_being_quantized); pool.install(|| { - for k in crate::vector::arroy_db_range_for_embedder(embedder_index) { - let mut writer = ArroyWrapper::new(vector_arroy, k, was_quantized); - if is_quantizing { - writer.quantize(wtxn, k, dimension)?; - } - if writer.need_build(wtxn, dimension)? { - writer.build(wtxn, &mut rng, dimension)?; - } else if writer.is_empty(wtxn, dimension)? { - break; - } - } + let mut writer = ArroyWrapper::new(vector_arroy, embedder_index, was_quantized); + writer.build_and_quantize(wtxn, &mut rng, dimension, is_quantizing, cancel)?; Result::Ok(()) }) .map_err(InternalError::from)??; diff --git a/milli/src/update/index_documents/parallel.rs b/crates/milli/src/update/index_documents/parallel.rs similarity index 100% rename from milli/src/update/index_documents/parallel.rs rename to crates/milli/src/update/index_documents/parallel.rs diff --git a/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/documents_ids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/documents_ids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/documents_ids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/documents_ids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/facet_id_exists_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/facet_id_exists_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/facet_id_exists_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/facet_id_exists_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/word_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/word_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/word_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/word_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/word_pair_proximity_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/word_pair_proximity_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/word_pair_proximity_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_numbers_as_primary_key/word_pair_proximity_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/documents_ids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/documents_ids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/documents_ids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/documents_ids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/word_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/word_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/word_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/word_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/word_pair_proximity_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/word_pair_proximity_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/word_pair_proximity_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/delete_documents_with_strange_primary_key/word_pair_proximity_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_exists_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_exists_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_exists_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_exists_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_f64_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_f64_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_f64_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_f64_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_string_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_string_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_string_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/facet_id_string_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/word_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/word_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/word_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/word_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/word_pair_proximity_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/word_pair_proximity_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/word_pair_proximity_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/filtered_placeholder_search_should_not_return_deleted_documents/word_pair_proximity_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/geo_filtered_placeholder_search_should_not_return_deleted_documents/facet_id_f64_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/geo_filtered_placeholder_search_should_not_return_deleted_documents/facet_id_f64_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/geo_filtered_placeholder_search_should_not_return_deleted_documents/facet_id_f64_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/geo_filtered_placeholder_search_should_not_return_deleted_documents/facet_id_f64_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/geo_filtered_placeholder_search_should_not_return_deleted_documents/facet_id_string_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/geo_filtered_placeholder_search_should_not_return_deleted_documents/facet_id_string_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/geo_filtered_placeholder_search_should_not_return_deleted_documents/facet_id_string_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/geo_filtered_placeholder_search_should_not_return_deleted_documents/facet_id_string_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/simple_documents_replace/initial/word_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/simple_documents_replace/initial/word_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/simple_documents_replace/initial/word_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/simple_documents_replace/initial/word_docids.snap diff --git a/milli/src/update/index_documents/snapshots/mod.rs/simple_documents_replace/updated/word_docids.snap b/crates/milli/src/update/index_documents/snapshots/mod.rs/simple_documents_replace/updated/word_docids.snap similarity index 100% rename from milli/src/update/index_documents/snapshots/mod.rs/simple_documents_replace/updated/word_docids.snap rename to crates/milli/src/update/index_documents/snapshots/mod.rs/simple_documents_replace/updated/word_docids.snap diff --git a/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs similarity index 96% rename from milli/src/update/index_documents/transform.rs rename to crates/milli/src/update/index_documents/transform.rs index 84135ff24..7239e8bff 100644 --- a/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -996,27 +996,24 @@ impl<'a, 'i> Transform<'a, 'i> { None }; - let readers: Result, &RoaringBitmap)>> = settings_diff + let readers: BTreeMap<&str, (ArroyWrapper, &RoaringBitmap)> = settings_diff .embedding_config_updates .iter() .filter_map(|(name, action)| { if let Some(WriteBackToDocuments { embedder_id, user_provided }) = action.write_back() { - let readers: Result> = self - .index - .arroy_readers(wtxn, *embedder_id, action.was_quantized) - .collect(); - match readers { - Ok(readers) => Some(Ok((name.as_str(), (readers, user_provided)))), - Err(error) => Some(Err(error)), - } + let reader = ArroyWrapper::new( + self.index.vector_arroy, + *embedder_id, + action.was_quantized, + ); + Some((name.as_str(), (reader, user_provided))) } else { None } }) .collect(); - let readers = readers?; let old_vectors_fid = settings_diff .old @@ -1055,34 +1052,24 @@ impl<'a, 'i> Transform<'a, 'i> { arroy::Error, > = readers .iter() - .filter_map(|(name, (readers, user_provided))| { + .filter_map(|(name, (reader, user_provided))| { if !user_provided.contains(docid) { return None; } - let mut vectors = Vec::new(); - for reader in readers { - let Some(vector) = reader.item_vector(wtxn, docid).transpose() else { - break; - }; - - match vector { - Ok(vector) => vectors.push(vector), - Err(error) => return Some(Err(error)), - } + match reader.item_vectors(wtxn, docid) { + Ok(vectors) if vectors.is_empty() => None, + Ok(vectors) => Some(Ok(( + name.to_string(), + serde_json::to_value(ExplicitVectors { + embeddings: Some( + VectorOrArrayOfVectors::from_array_of_vectors(vectors), + ), + regenerate: false, + }) + .unwrap(), + ))), + Err(e) => Some(Err(e)), } - if vectors.is_empty() { - return None; - } - Some(Ok(( - name.to_string(), - serde_json::to_value(ExplicitVectors { - embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors( - vectors, - )), - regenerate: false, - }) - .unwrap(), - ))) }) .collect(); @@ -1111,11 +1098,9 @@ impl<'a, 'i> Transform<'a, 'i> { } // delete all vectors from the embedders that need removal - for (_, (readers, _)) in readers { - for reader in readers { - let dimensions = reader.dimensions(wtxn)?; - reader.clear(wtxn, dimensions)?; - } + for (_, (reader, _)) in readers { + let dimensions = reader.dimensions(wtxn)?; + reader.clear(wtxn, dimensions)?; } let grenad_params = GrenadParameters { diff --git a/milli/src/update/index_documents/typed_chunk.rs b/crates/milli/src/update/index_documents/typed_chunk.rs similarity index 92% rename from milli/src/update/index_documents/typed_chunk.rs rename to crates/milli/src/update/index_documents/typed_chunk.rs index 1f477cf4b..2c30220bc 100644 --- a/milli/src/update/index_documents/typed_chunk.rs +++ b/crates/milli/src/update/index_documents/typed_chunk.rs @@ -668,22 +668,14 @@ pub(crate) fn write_typed_chunk_into_index( .get(&embedder_name) .map_or(false, |conf| conf.2); // FIXME: allow customizing distance - let writers: Vec<_> = crate::vector::arroy_db_range_for_embedder(embedder_index) - .map(|k| ArroyWrapper::new(index.vector_arroy, k, binary_quantized)) - .collect(); + let writer = ArroyWrapper::new(index.vector_arroy, embedder_index, binary_quantized); // remove vectors for docids we want them removed let merger = remove_vectors_builder.build(); let mut iter = merger.into_stream_merger_iter()?; while let Some((key, _)) = iter.next()? { let docid = key.try_into().map(DocumentId::from_be_bytes).unwrap(); - - for writer in &writers { - // Uses invariant: vectors are packed in the first writers. - if !writer.del_item(wtxn, expected_dimension, docid)? { - break; - } - } + writer.del_items(wtxn, expected_dimension, docid)?; } // add generated embeddings @@ -711,9 +703,7 @@ pub(crate) fn write_typed_chunk_into_index( embeddings.embedding_count(), ))); } - for (embedding, writer) in embeddings.iter().zip(&writers) { - writer.add_item(wtxn, expected_dimension, docid, embedding)?; - } + writer.add_items(wtxn, docid, &embeddings)?; } // perform the manual diff @@ -728,51 +718,14 @@ pub(crate) fn write_typed_chunk_into_index( if let Some(value) = vector_deladd_obkv.get(DelAdd::Deletion) { let vector: Vec = pod_collect_to_vec(value); - let mut deleted_index = None; - for (index, writer) in writers.iter().enumerate() { - let Some(candidate) = writer.item_vector(wtxn, docid)? else { - // uses invariant: vectors are packed in the first writers. - break; - }; - if candidate == vector { - writer.del_item(wtxn, expected_dimension, docid)?; - deleted_index = Some(index); - } - } - - // 🥲 enforce invariant: vectors are packed in the first writers. - if let Some(deleted_index) = deleted_index { - let mut last_index_with_a_vector = None; - for (index, writer) in writers.iter().enumerate().skip(deleted_index) { - let Some(candidate) = writer.item_vector(wtxn, docid)? else { - break; - }; - last_index_with_a_vector = Some((index, candidate)); - } - if let Some((last_index, vector)) = last_index_with_a_vector { - // unwrap: computed the index from the list of writers - let writer = writers.get(last_index).unwrap(); - writer.del_item(wtxn, expected_dimension, docid)?; - writers.get(deleted_index).unwrap().add_item( - wtxn, - expected_dimension, - docid, - &vector, - )?; - } - } + writer.del_item(wtxn, docid, &vector)?; } if let Some(value) = vector_deladd_obkv.get(DelAdd::Addition) { let vector = pod_collect_to_vec(value); // overflow was detected during vector extraction. - for writer in &writers { - if !writer.contains_item(wtxn, expected_dimension, docid)? { - writer.add_item(wtxn, expected_dimension, docid, &vector)?; - break; - } - } + writer.add_item(wtxn, docid, &vector)?; } } diff --git a/milli/src/update/indexer_config.rs b/crates/milli/src/update/indexer_config.rs similarity index 100% rename from milli/src/update/indexer_config.rs rename to crates/milli/src/update/indexer_config.rs diff --git a/milli/src/update/mod.rs b/crates/milli/src/update/mod.rs similarity index 100% rename from milli/src/update/mod.rs rename to crates/milli/src/update/mod.rs diff --git a/milli/src/update/new/channel.rs b/crates/milli/src/update/new/channel.rs similarity index 100% rename from milli/src/update/new/channel.rs rename to crates/milli/src/update/new/channel.rs diff --git a/milli/src/update/new/document.rs b/crates/milli/src/update/new/document.rs similarity index 100% rename from milli/src/update/new/document.rs rename to crates/milli/src/update/new/document.rs diff --git a/milli/src/update/new/document_change.rs b/crates/milli/src/update/new/document_change.rs similarity index 100% rename from milli/src/update/new/document_change.rs rename to crates/milli/src/update/new/document_change.rs diff --git a/milli/src/update/new/extract/cache.rs b/crates/milli/src/update/new/extract/cache.rs similarity index 100% rename from milli/src/update/new/extract/cache.rs rename to crates/milli/src/update/new/extract/cache.rs diff --git a/milli/src/update/new/extract/documents.rs b/crates/milli/src/update/new/extract/documents.rs similarity index 100% rename from milli/src/update/new/extract/documents.rs rename to crates/milli/src/update/new/extract/documents.rs diff --git a/milli/src/update/new/extract/faceted/extract_facets.rs b/crates/milli/src/update/new/extract/faceted/extract_facets.rs similarity index 100% rename from milli/src/update/new/extract/faceted/extract_facets.rs rename to crates/milli/src/update/new/extract/faceted/extract_facets.rs diff --git a/milli/src/update/new/extract/faceted/facet_document.rs b/crates/milli/src/update/new/extract/faceted/facet_document.rs similarity index 100% rename from milli/src/update/new/extract/faceted/facet_document.rs rename to crates/milli/src/update/new/extract/faceted/facet_document.rs diff --git a/milli/src/update/new/extract/faceted/mod.rs b/crates/milli/src/update/new/extract/faceted/mod.rs similarity index 100% rename from milli/src/update/new/extract/faceted/mod.rs rename to crates/milli/src/update/new/extract/faceted/mod.rs diff --git a/milli/src/update/new/extract/mod.rs b/crates/milli/src/update/new/extract/mod.rs similarity index 100% rename from milli/src/update/new/extract/mod.rs rename to crates/milli/src/update/new/extract/mod.rs diff --git a/milli/src/update/new/extract/searchable/extract_word_docids.rs b/crates/milli/src/update/new/extract/searchable/extract_word_docids.rs similarity index 100% rename from milli/src/update/new/extract/searchable/extract_word_docids.rs rename to crates/milli/src/update/new/extract/searchable/extract_word_docids.rs diff --git a/milli/src/update/new/extract/searchable/extract_word_pair_proximity_docids.rs b/crates/milli/src/update/new/extract/searchable/extract_word_pair_proximity_docids.rs similarity index 100% rename from milli/src/update/new/extract/searchable/extract_word_pair_proximity_docids.rs rename to crates/milli/src/update/new/extract/searchable/extract_word_pair_proximity_docids.rs diff --git a/milli/src/update/new/extract/searchable/mod.rs b/crates/milli/src/update/new/extract/searchable/mod.rs similarity index 100% rename from milli/src/update/new/extract/searchable/mod.rs rename to crates/milli/src/update/new/extract/searchable/mod.rs diff --git a/milli/src/update/new/extract/searchable/tokenize_document.rs b/crates/milli/src/update/new/extract/searchable/tokenize_document.rs similarity index 100% rename from milli/src/update/new/extract/searchable/tokenize_document.rs rename to crates/milli/src/update/new/extract/searchable/tokenize_document.rs diff --git a/milli/src/update/new/facet_search_builder.rs b/crates/milli/src/update/new/facet_search_builder.rs similarity index 100% rename from milli/src/update/new/facet_search_builder.rs rename to crates/milli/src/update/new/facet_search_builder.rs diff --git a/milli/src/update/new/fst_merger_builder.rs b/crates/milli/src/update/new/fst_merger_builder.rs similarity index 100% rename from milli/src/update/new/fst_merger_builder.rs rename to crates/milli/src/update/new/fst_merger_builder.rs diff --git a/milli/src/update/new/indexer/de.rs b/crates/milli/src/update/new/indexer/de.rs similarity index 100% rename from milli/src/update/new/indexer/de.rs rename to crates/milli/src/update/new/indexer/de.rs diff --git a/milli/src/update/new/indexer/document_changes.rs b/crates/milli/src/update/new/indexer/document_changes.rs similarity index 100% rename from milli/src/update/new/indexer/document_changes.rs rename to crates/milli/src/update/new/indexer/document_changes.rs diff --git a/milli/src/update/new/indexer/document_deletion.rs b/crates/milli/src/update/new/indexer/document_deletion.rs similarity index 100% rename from milli/src/update/new/indexer/document_deletion.rs rename to crates/milli/src/update/new/indexer/document_deletion.rs diff --git a/milli/src/update/new/indexer/document_operation.rs b/crates/milli/src/update/new/indexer/document_operation.rs similarity index 100% rename from milli/src/update/new/indexer/document_operation.rs rename to crates/milli/src/update/new/indexer/document_operation.rs diff --git a/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs similarity index 100% rename from milli/src/update/new/indexer/mod.rs rename to crates/milli/src/update/new/indexer/mod.rs diff --git a/milli/src/update/new/indexer/partial_dump.rs b/crates/milli/src/update/new/indexer/partial_dump.rs similarity index 100% rename from milli/src/update/new/indexer/partial_dump.rs rename to crates/milli/src/update/new/indexer/partial_dump.rs diff --git a/milli/src/update/new/indexer/update_by_function.rs b/crates/milli/src/update/new/indexer/update_by_function.rs similarity index 100% rename from milli/src/update/new/indexer/update_by_function.rs rename to crates/milli/src/update/new/indexer/update_by_function.rs diff --git a/milli/src/update/new/merger.rs b/crates/milli/src/update/new/merger.rs similarity index 100% rename from milli/src/update/new/merger.rs rename to crates/milli/src/update/new/merger.rs diff --git a/milli/src/update/new/mod.rs b/crates/milli/src/update/new/mod.rs similarity index 100% rename from milli/src/update/new/mod.rs rename to crates/milli/src/update/new/mod.rs diff --git a/milli/src/update/new/parallel_iterator_ext.rs b/crates/milli/src/update/new/parallel_iterator_ext.rs similarity index 100% rename from milli/src/update/new/parallel_iterator_ext.rs rename to crates/milli/src/update/new/parallel_iterator_ext.rs diff --git a/milli/src/update/new/top_level_map.rs b/crates/milli/src/update/new/top_level_map.rs similarity index 100% rename from milli/src/update/new/top_level_map.rs rename to crates/milli/src/update/new/top_level_map.rs diff --git a/milli/src/update/new/word_fst_builder.rs b/crates/milli/src/update/new/word_fst_builder.rs similarity index 100% rename from milli/src/update/new/word_fst_builder.rs rename to crates/milli/src/update/new/word_fst_builder.rs diff --git a/milli/src/update/new/words_prefix_docids.rs b/crates/milli/src/update/new/words_prefix_docids.rs similarity index 100% rename from milli/src/update/new/words_prefix_docids.rs rename to crates/milli/src/update/new/words_prefix_docids.rs diff --git a/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs similarity index 100% rename from milli/src/update/settings.rs rename to crates/milli/src/update/settings.rs diff --git a/milli/src/update/update_step.rs b/crates/milli/src/update/update_step.rs similarity index 100% rename from milli/src/update/update_step.rs rename to crates/milli/src/update/update_step.rs diff --git a/milli/src/update/word_prefix_docids.rs b/crates/milli/src/update/word_prefix_docids.rs similarity index 100% rename from milli/src/update/word_prefix_docids.rs rename to crates/milli/src/update/word_prefix_docids.rs diff --git a/milli/src/update/words_prefix_integer_docids.rs b/crates/milli/src/update/words_prefix_integer_docids.rs similarity index 100% rename from milli/src/update/words_prefix_integer_docids.rs rename to crates/milli/src/update/words_prefix_integer_docids.rs diff --git a/milli/src/update/words_prefixes_fst.rs b/crates/milli/src/update/words_prefixes_fst.rs similarity index 100% rename from milli/src/update/words_prefixes_fst.rs rename to crates/milli/src/update/words_prefixes_fst.rs diff --git a/milli/src/vector/error.rs b/crates/milli/src/vector/error.rs similarity index 100% rename from milli/src/vector/error.rs rename to crates/milli/src/vector/error.rs diff --git a/milli/src/vector/hf.rs b/crates/milli/src/vector/hf.rs similarity index 100% rename from milli/src/vector/hf.rs rename to crates/milli/src/vector/hf.rs diff --git a/milli/src/vector/json_template.rs b/crates/milli/src/vector/json_template.rs similarity index 100% rename from milli/src/vector/json_template.rs rename to crates/milli/src/vector/json_template.rs diff --git a/milli/src/vector/manual.rs b/crates/milli/src/vector/manual.rs similarity index 100% rename from milli/src/vector/manual.rs rename to crates/milli/src/vector/manual.rs diff --git a/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs similarity index 60% rename from milli/src/vector/mod.rs rename to crates/milli/src/vector/mod.rs index d52e68bbe..571c02c8c 100644 --- a/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; -use arroy::distances::{Angular, BinaryQuantizedAngular}; +use arroy::distances::{BinaryQuantizedCosine, Cosine}; use arroy::ItemId; use deserr::{DeserializeError, Deserr}; use heed::{RoTxn, RwTxn, Unspecified}; @@ -32,105 +32,243 @@ pub const REQUEST_PARALLELISM: usize = 40; pub struct ArroyWrapper { quantized: bool, - index: u16, + embedder_index: u8, database: arroy::Database, } impl ArroyWrapper { - pub fn new(database: arroy::Database, index: u16, quantized: bool) -> Self { - Self { database, index, quantized } + pub fn new( + database: arroy::Database, + embedder_index: u8, + quantized: bool, + ) -> Self { + Self { database, embedder_index, quantized } } - pub fn index(&self) -> u16 { - self.index + pub fn embedder_index(&self) -> u8 { + self.embedder_index + } + + fn readers<'a, D: arroy::Distance>( + &'a self, + rtxn: &'a RoTxn<'a>, + db: arroy::Database, + ) -> impl Iterator, arroy::Error>> + 'a { + arroy_db_range_for_embedder(self.embedder_index).map_while(move |index| { + match arroy::Reader::open(rtxn, index, db) { + Ok(reader) => match reader.is_empty(rtxn) { + Ok(false) => Some(Ok(reader)), + Ok(true) => None, + Err(e) => Some(Err(e)), + }, + Err(arroy::Error::MissingMetadata(_)) => None, + Err(e) => Some(Err(e)), + } + }) } pub fn dimensions(&self, rtxn: &RoTxn) -> Result { + let first_id = arroy_db_range_for_embedder(self.embedder_index).next().unwrap(); if self.quantized { - Ok(arroy::Reader::open(rtxn, self.index, self.quantized_db())?.dimensions()) + Ok(arroy::Reader::open(rtxn, first_id, self.quantized_db())?.dimensions()) } else { - Ok(arroy::Reader::open(rtxn, self.index, self.angular_db())?.dimensions()) + Ok(arroy::Reader::open(rtxn, first_id, self.angular_db())?.dimensions()) } } - pub fn quantize( + pub fn build_and_quantize( &mut self, wtxn: &mut RwTxn, - index: u16, + rng: &mut R, dimension: usize, + quantizing: bool, + cancel: &(impl Fn() -> bool + Sync + Send), ) -> Result<(), arroy::Error> { - if !self.quantized { - let writer = arroy::Writer::new(self.angular_db(), index, dimension); - writer.prepare_changing_distance::(wtxn)?; - self.quantized = true; + for index in arroy_db_range_for_embedder(self.embedder_index) { + if self.quantized { + let writer = arroy::Writer::new(self.quantized_db(), index, dimension); + if writer.need_build(wtxn)? { + writer.builder(rng).build(wtxn)? + } else if writer.is_empty(wtxn)? { + break; + } + } else { + let writer = arroy::Writer::new(self.angular_db(), index, dimension); + // If we are quantizing the databases, we can't know from meilisearch + // if the db was empty but still contained the wrong metadata, thus we need + // to quantize everything and can't stop early. Since this operation can + // only happens once in the life of an embedder, it's not very performances + // sensitive. + if quantizing && !self.quantized { + let writer = writer.prepare_changing_distance::(wtxn)?; + writer.builder(rng).cancel(cancel).build(wtxn)?; + } else if writer.need_build(wtxn)? { + writer.builder(rng).cancel(cancel).build(wtxn)?; + } else if writer.is_empty(wtxn)? { + break; + } + } } Ok(()) } - pub fn need_build(&self, rtxn: &RoTxn, dimension: usize) -> Result { - if self.quantized { - arroy::Writer::new(self.quantized_db(), self.index, dimension).need_build(rtxn) - } else { - arroy::Writer::new(self.angular_db(), self.index, dimension).need_build(rtxn) - } - } - - pub fn build( + /// Overwrite all the embeddings associated with the index and item ID. + /// /!\ It won't remove embeddings after the last passed embedding, which can leave stale embeddings. + /// You should call `del_items` on the `item_id` before calling this method. + /// /!\ Cannot insert more than u8::MAX embeddings; after inserting u8::MAX embeddings, all the remaining ones will be silently ignored. + pub fn add_items( &self, wtxn: &mut RwTxn, - rng: &mut R, - dimension: usize, + item_id: arroy::ItemId, + embeddings: &Embeddings, ) -> Result<(), arroy::Error> { - if self.quantized { - arroy::Writer::new(self.quantized_db(), self.index, dimension).build(wtxn, rng, None) - } else { - arroy::Writer::new(self.angular_db(), self.index, dimension).build(wtxn, rng, None) + let dimension = embeddings.dimension(); + for (index, vector) in + arroy_db_range_for_embedder(self.embedder_index).zip(embeddings.iter()) + { + if self.quantized { + arroy::Writer::new(self.quantized_db(), index, dimension) + .add_item(wtxn, item_id, vector)? + } else { + arroy::Writer::new(self.angular_db(), index, dimension) + .add_item(wtxn, item_id, vector)? + } } + Ok(()) } + /// Add one document int for this index where we can find an empty spot. pub fn add_item( &self, wtxn: &mut RwTxn, - dimension: usize, item_id: arroy::ItemId, vector: &[f32], ) -> Result<(), arroy::Error> { if self.quantized { - arroy::Writer::new(self.quantized_db(), self.index, dimension) - .add_item(wtxn, item_id, vector) + self._add_item(wtxn, self.quantized_db(), item_id, vector) } else { - arroy::Writer::new(self.angular_db(), self.index, dimension) - .add_item(wtxn, item_id, vector) + self._add_item(wtxn, self.angular_db(), item_id, vector) } } - pub fn del_item( + fn _add_item( + &self, + wtxn: &mut RwTxn, + db: arroy::Database, + item_id: arroy::ItemId, + vector: &[f32], + ) -> Result<(), arroy::Error> { + let dimension = vector.len(); + + for index in arroy_db_range_for_embedder(self.embedder_index) { + let writer = arroy::Writer::new(db, index, dimension); + if !writer.contains_item(wtxn, item_id)? { + writer.add_item(wtxn, item_id, vector)?; + break; + } + } + Ok(()) + } + + /// Delete all embeddings from a specific `item_id` + pub fn del_items( &self, wtxn: &mut RwTxn, dimension: usize, item_id: arroy::ItemId, + ) -> Result<(), arroy::Error> { + for index in arroy_db_range_for_embedder(self.embedder_index) { + if self.quantized { + let writer = arroy::Writer::new(self.quantized_db(), index, dimension); + if !writer.del_item(wtxn, item_id)? { + break; + } + } else { + let writer = arroy::Writer::new(self.angular_db(), index, dimension); + if !writer.del_item(wtxn, item_id)? { + break; + } + } + } + + Ok(()) + } + + /// Delete one item. + pub fn del_item( + &self, + wtxn: &mut RwTxn, + item_id: arroy::ItemId, + vector: &[f32], ) -> Result { if self.quantized { - arroy::Writer::new(self.quantized_db(), self.index, dimension).del_item(wtxn, item_id) + self._del_item(wtxn, self.quantized_db(), item_id, vector) } else { - arroy::Writer::new(self.angular_db(), self.index, dimension).del_item(wtxn, item_id) + self._del_item(wtxn, self.angular_db(), item_id, vector) } } + fn _del_item( + &self, + wtxn: &mut RwTxn, + db: arroy::Database, + item_id: arroy::ItemId, + vector: &[f32], + ) -> Result { + let dimension = vector.len(); + let mut deleted_index = None; + + for index in arroy_db_range_for_embedder(self.embedder_index) { + let writer = arroy::Writer::new(db, index, dimension); + let Some(candidate) = writer.item_vector(wtxn, item_id)? else { + // uses invariant: vectors are packed in the first writers. + break; + }; + if candidate == vector { + writer.del_item(wtxn, item_id)?; + deleted_index = Some(index); + } + } + + // 🥲 enforce invariant: vectors are packed in the first writers. + if let Some(deleted_index) = deleted_index { + let mut last_index_with_a_vector = None; + for index in + arroy_db_range_for_embedder(self.embedder_index).skip(deleted_index as usize) + { + let writer = arroy::Writer::new(db, index, dimension); + let Some(candidate) = writer.item_vector(wtxn, item_id)? else { + break; + }; + last_index_with_a_vector = Some((index, candidate)); + } + if let Some((last_index, vector)) = last_index_with_a_vector { + let writer = arroy::Writer::new(db, last_index, dimension); + writer.del_item(wtxn, item_id)?; + let writer = arroy::Writer::new(db, deleted_index, dimension); + writer.add_item(wtxn, item_id, &vector)?; + } + } + Ok(deleted_index.is_some()) + } + pub fn clear(&self, wtxn: &mut RwTxn, dimension: usize) -> Result<(), arroy::Error> { - if self.quantized { - arroy::Writer::new(self.quantized_db(), self.index, dimension).clear(wtxn) - } else { - arroy::Writer::new(self.angular_db(), self.index, dimension).clear(wtxn) - } - } - - pub fn is_empty(&self, rtxn: &RoTxn, dimension: usize) -> Result { - if self.quantized { - arroy::Writer::new(self.quantized_db(), self.index, dimension).is_empty(rtxn) - } else { - arroy::Writer::new(self.angular_db(), self.index, dimension).is_empty(rtxn) + for index in arroy_db_range_for_embedder(self.embedder_index) { + if self.quantized { + let writer = arroy::Writer::new(self.quantized_db(), index, dimension); + if writer.is_empty(wtxn)? { + break; + } + writer.clear(wtxn)?; + } else { + let writer = arroy::Writer::new(self.angular_db(), index, dimension); + if writer.is_empty(wtxn)? { + break; + } + writer.clear(wtxn)?; + } } + Ok(()) } pub fn contains_item( @@ -139,11 +277,25 @@ impl ArroyWrapper { dimension: usize, item: arroy::ItemId, ) -> Result { - if self.quantized { - arroy::Writer::new(self.quantized_db(), self.index, dimension).contains_item(rtxn, item) - } else { - arroy::Writer::new(self.angular_db(), self.index, dimension).contains_item(rtxn, item) + for index in arroy_db_range_for_embedder(self.embedder_index) { + let contains = if self.quantized { + let writer = arroy::Writer::new(self.quantized_db(), index, dimension); + if writer.is_empty(rtxn)? { + break; + } + writer.contains_item(rtxn, item)? + } else { + let writer = arroy::Writer::new(self.angular_db(), index, dimension); + if writer.is_empty(rtxn)? { + break; + } + writer.contains_item(rtxn, item)? + }; + if contains { + return Ok(contains); + } } + Ok(false) } pub fn nns_by_item( @@ -152,45 +304,108 @@ impl ArroyWrapper { item: ItemId, limit: usize, filter: Option<&RoaringBitmap>, - ) -> Result>, arroy::Error> { + ) -> Result, arroy::Error> { if self.quantized { - arroy::Reader::open(rtxn, self.index, self.quantized_db())? - .nns_by_item(rtxn, item, limit, None, None, filter) + self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter) } else { - arroy::Reader::open(rtxn, self.index, self.angular_db())? - .nns_by_item(rtxn, item, limit, None, None, filter) + self._nns_by_item(rtxn, self.angular_db(), item, limit, filter) } } + fn _nns_by_item( + &self, + rtxn: &RoTxn, + db: arroy::Database, + item: ItemId, + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> Result, arroy::Error> { + let mut results = Vec::new(); + + for reader in self.readers(rtxn, db) { + let reader = reader?; + let mut searcher = reader.nns(limit); + if let Some(filter) = filter { + searcher.candidates(filter); + } + + if let Some(mut ret) = searcher.by_item(rtxn, item)? { + results.append(&mut ret); + } else { + break; + } + } + results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + Ok(results) + } + pub fn nns_by_vector( &self, - txn: &RoTxn, - item: &[f32], + rtxn: &RoTxn, + vector: &[f32], limit: usize, filter: Option<&RoaringBitmap>, ) -> Result, arroy::Error> { if self.quantized { - arroy::Reader::open(txn, self.index, self.quantized_db())? - .nns_by_vector(txn, item, limit, None, None, filter) + self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) } else { - arroy::Reader::open(txn, self.index, self.angular_db())? - .nns_by_vector(txn, item, limit, None, None, filter) + self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter) } } - pub fn item_vector(&self, rtxn: &RoTxn, docid: u32) -> Result>, arroy::Error> { + fn _nns_by_vector( + &self, + rtxn: &RoTxn, + db: arroy::Database, + vector: &[f32], + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> Result, arroy::Error> { + let mut results = Vec::new(); + + for reader in self.readers(rtxn, db) { + let reader = reader?; + let mut searcher = reader.nns(limit); + if let Some(filter) = filter { + searcher.candidates(filter); + } + + results.append(&mut searcher.by_vector(rtxn, vector)?); + } + + results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + + Ok(results) + } + + pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> Result>, arroy::Error> { + let mut vectors = Vec::new(); + if self.quantized { - arroy::Reader::open(rtxn, self.index, self.quantized_db())?.item_vector(rtxn, docid) + for reader in self.readers(rtxn, self.quantized_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } else { + break; + } + } } else { - arroy::Reader::open(rtxn, self.index, self.angular_db())?.item_vector(rtxn, docid) + for reader in self.readers(rtxn, self.angular_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } else { + break; + } + } } + Ok(vectors) } - fn angular_db(&self) -> arroy::Database { + fn angular_db(&self) -> arroy::Database { self.database.remap_data_type() } - fn quantized_db(&self) -> arroy::Database { + fn quantized_db(&self) -> arroy::Database { self.database.remap_data_type() } } diff --git a/milli/src/vector/ollama.rs b/crates/milli/src/vector/ollama.rs similarity index 100% rename from milli/src/vector/ollama.rs rename to crates/milli/src/vector/ollama.rs diff --git a/milli/src/vector/openai.rs b/crates/milli/src/vector/openai.rs similarity index 100% rename from milli/src/vector/openai.rs rename to crates/milli/src/vector/openai.rs diff --git a/milli/src/vector/parsed_vectors.rs b/crates/milli/src/vector/parsed_vectors.rs similarity index 100% rename from milli/src/vector/parsed_vectors.rs rename to crates/milli/src/vector/parsed_vectors.rs diff --git a/milli/src/vector/rest.rs b/crates/milli/src/vector/rest.rs similarity index 100% rename from milli/src/vector/rest.rs rename to crates/milli/src/vector/rest.rs diff --git a/milli/src/vector/settings.rs b/crates/milli/src/vector/settings.rs similarity index 97% rename from milli/src/vector/settings.rs rename to crates/milli/src/vector/settings.rs index 3bb7f09e6..d1cf364a2 100644 --- a/milli/src/vector/settings.rs +++ b/crates/milli/src/vector/settings.rs @@ -417,6 +417,8 @@ impl EmbeddingSettings { pub const DISTRIBUTION: &'static str = "distribution"; + pub const BINARY_QUANTIZED: &'static str = "binaryQuantized"; + pub fn allowed_sources_for_field(field: &'static str) -> &'static [EmbedderSource] { match field { Self::SOURCE => &[ @@ -456,6 +458,13 @@ impl EmbeddingSettings { EmbedderSource::Rest, EmbedderSource::UserProvided, ], + Self::BINARY_QUANTIZED => &[ + EmbedderSource::HuggingFace, + EmbedderSource::Ollama, + EmbedderSource::OpenAi, + EmbedderSource::Rest, + EmbedderSource::UserProvided, + ], _other => unreachable!("unknown field"), } } @@ -470,6 +479,7 @@ impl EmbeddingSettings { Self::DIMENSIONS, Self::DISTRIBUTION, Self::URL, + Self::BINARY_QUANTIZED, ], EmbedderSource::HuggingFace => &[ Self::SOURCE, @@ -477,6 +487,7 @@ impl EmbeddingSettings { Self::REVISION, Self::DOCUMENT_TEMPLATE, Self::DISTRIBUTION, + Self::BINARY_QUANTIZED, ], EmbedderSource::Ollama => &[ Self::SOURCE, @@ -486,8 +497,11 @@ impl EmbeddingSettings { Self::API_KEY, Self::DIMENSIONS, Self::DISTRIBUTION, + Self::BINARY_QUANTIZED, ], - EmbedderSource::UserProvided => &[Self::SOURCE, Self::DIMENSIONS, Self::DISTRIBUTION], + EmbedderSource::UserProvided => { + &[Self::SOURCE, Self::DIMENSIONS, Self::DISTRIBUTION, Self::BINARY_QUANTIZED] + } EmbedderSource::Rest => &[ Self::SOURCE, Self::API_KEY, @@ -498,6 +512,7 @@ impl EmbeddingSettings { Self::RESPONSE, Self::HEADERS, Self::DISTRIBUTION, + Self::BINARY_QUANTIZED, ], } } diff --git a/milli/tests/assets/test_set.ndjson b/crates/milli/tests/assets/test_set.ndjson similarity index 100% rename from milli/tests/assets/test_set.ndjson rename to crates/milli/tests/assets/test_set.ndjson diff --git a/milli/tests/mod.rs b/crates/milli/tests/mod.rs similarity index 100% rename from milli/tests/mod.rs rename to crates/milli/tests/mod.rs diff --git a/milli/tests/search/distinct.rs b/crates/milli/tests/search/distinct.rs similarity index 100% rename from milli/tests/search/distinct.rs rename to crates/milli/tests/search/distinct.rs diff --git a/milli/tests/search/facet_distribution.rs b/crates/milli/tests/search/facet_distribution.rs similarity index 100% rename from milli/tests/search/facet_distribution.rs rename to crates/milli/tests/search/facet_distribution.rs diff --git a/milli/tests/search/filters.rs b/crates/milli/tests/search/filters.rs similarity index 100% rename from milli/tests/search/filters.rs rename to crates/milli/tests/search/filters.rs diff --git a/milli/tests/search/mod.rs b/crates/milli/tests/search/mod.rs similarity index 100% rename from milli/tests/search/mod.rs rename to crates/milli/tests/search/mod.rs diff --git a/milli/tests/search/phrase_search.rs b/crates/milli/tests/search/phrase_search.rs similarity index 100% rename from milli/tests/search/phrase_search.rs rename to crates/milli/tests/search/phrase_search.rs diff --git a/milli/tests/search/query_criteria.rs b/crates/milli/tests/search/query_criteria.rs similarity index 100% rename from milli/tests/search/query_criteria.rs rename to crates/milli/tests/search/query_criteria.rs diff --git a/milli/tests/search/sort.rs b/crates/milli/tests/search/sort.rs similarity index 100% rename from milli/tests/search/sort.rs rename to crates/milli/tests/search/sort.rs diff --git a/milli/tests/search/typo_tolerance.rs b/crates/milli/tests/search/typo_tolerance.rs similarity index 100% rename from milli/tests/search/typo_tolerance.rs rename to crates/milli/tests/search/typo_tolerance.rs diff --git a/permissive-json-pointer/Cargo.toml b/crates/permissive-json-pointer/Cargo.toml similarity index 100% rename from permissive-json-pointer/Cargo.toml rename to crates/permissive-json-pointer/Cargo.toml diff --git a/permissive-json-pointer/README.md b/crates/permissive-json-pointer/README.md similarity index 100% rename from permissive-json-pointer/README.md rename to crates/permissive-json-pointer/README.md diff --git a/permissive-json-pointer/src/lib.rs b/crates/permissive-json-pointer/src/lib.rs similarity index 100% rename from permissive-json-pointer/src/lib.rs rename to crates/permissive-json-pointer/src/lib.rs diff --git a/tracing-trace/Cargo.toml b/crates/tracing-trace/Cargo.toml similarity index 100% rename from tracing-trace/Cargo.toml rename to crates/tracing-trace/Cargo.toml diff --git a/tracing-trace/src/bin/trace-to-callstats.rs b/crates/tracing-trace/src/bin/trace-to-callstats.rs similarity index 100% rename from tracing-trace/src/bin/trace-to-callstats.rs rename to crates/tracing-trace/src/bin/trace-to-callstats.rs diff --git a/tracing-trace/src/bin/trace-to-firefox.rs b/crates/tracing-trace/src/bin/trace-to-firefox.rs similarity index 100% rename from tracing-trace/src/bin/trace-to-firefox.rs rename to crates/tracing-trace/src/bin/trace-to-firefox.rs diff --git a/tracing-trace/src/entry.rs b/crates/tracing-trace/src/entry.rs similarity index 100% rename from tracing-trace/src/entry.rs rename to crates/tracing-trace/src/entry.rs diff --git a/tracing-trace/src/error.rs b/crates/tracing-trace/src/error.rs similarity index 100% rename from tracing-trace/src/error.rs rename to crates/tracing-trace/src/error.rs diff --git a/tracing-trace/src/layer.rs b/crates/tracing-trace/src/layer.rs similarity index 100% rename from tracing-trace/src/layer.rs rename to crates/tracing-trace/src/layer.rs diff --git a/tracing-trace/src/lib.rs b/crates/tracing-trace/src/lib.rs similarity index 100% rename from tracing-trace/src/lib.rs rename to crates/tracing-trace/src/lib.rs diff --git a/tracing-trace/src/main.rs b/crates/tracing-trace/src/main.rs similarity index 100% rename from tracing-trace/src/main.rs rename to crates/tracing-trace/src/main.rs diff --git a/tracing-trace/src/processor/firefox_profiler.rs b/crates/tracing-trace/src/processor/firefox_profiler.rs similarity index 100% rename from tracing-trace/src/processor/firefox_profiler.rs rename to crates/tracing-trace/src/processor/firefox_profiler.rs diff --git a/tracing-trace/src/processor/fmt.rs b/crates/tracing-trace/src/processor/fmt.rs similarity index 100% rename from tracing-trace/src/processor/fmt.rs rename to crates/tracing-trace/src/processor/fmt.rs diff --git a/tracing-trace/src/processor/mod.rs b/crates/tracing-trace/src/processor/mod.rs similarity index 100% rename from tracing-trace/src/processor/mod.rs rename to crates/tracing-trace/src/processor/mod.rs diff --git a/tracing-trace/src/processor/span_stats.rs b/crates/tracing-trace/src/processor/span_stats.rs similarity index 100% rename from tracing-trace/src/processor/span_stats.rs rename to crates/tracing-trace/src/processor/span_stats.rs diff --git a/xtask/Cargo.toml b/crates/xtask/Cargo.toml similarity index 100% rename from xtask/Cargo.toml rename to crates/xtask/Cargo.toml diff --git a/xtask/src/bench/assets.rs b/crates/xtask/src/bench/assets.rs similarity index 100% rename from xtask/src/bench/assets.rs rename to crates/xtask/src/bench/assets.rs diff --git a/xtask/src/bench/client.rs b/crates/xtask/src/bench/client.rs similarity index 100% rename from xtask/src/bench/client.rs rename to crates/xtask/src/bench/client.rs diff --git a/xtask/src/bench/command.rs b/crates/xtask/src/bench/command.rs similarity index 100% rename from xtask/src/bench/command.rs rename to crates/xtask/src/bench/command.rs diff --git a/xtask/src/bench/dashboard.rs b/crates/xtask/src/bench/dashboard.rs similarity index 100% rename from xtask/src/bench/dashboard.rs rename to crates/xtask/src/bench/dashboard.rs diff --git a/xtask/src/bench/env_info.rs b/crates/xtask/src/bench/env_info.rs similarity index 100% rename from xtask/src/bench/env_info.rs rename to crates/xtask/src/bench/env_info.rs diff --git a/xtask/src/bench/meili_process.rs b/crates/xtask/src/bench/meili_process.rs similarity index 100% rename from xtask/src/bench/meili_process.rs rename to crates/xtask/src/bench/meili_process.rs diff --git a/xtask/src/bench/mod.rs b/crates/xtask/src/bench/mod.rs similarity index 100% rename from xtask/src/bench/mod.rs rename to crates/xtask/src/bench/mod.rs diff --git a/xtask/src/bench/workload.rs b/crates/xtask/src/bench/workload.rs similarity index 100% rename from xtask/src/bench/workload.rs rename to crates/xtask/src/bench/workload.rs diff --git a/xtask/src/lib.rs b/crates/xtask/src/lib.rs similarity index 100% rename from xtask/src/lib.rs rename to crates/xtask/src/lib.rs diff --git a/xtask/src/main.rs b/crates/xtask/src/main.rs similarity index 100% rename from xtask/src/main.rs rename to crates/xtask/src/main.rs diff --git a/meilisearch/src/analytics/mod.rs b/meilisearch/src/analytics/mod.rs deleted file mode 100644 index 3c7ca0ed3..000000000 --- a/meilisearch/src/analytics/mod.rs +++ /dev/null @@ -1,137 +0,0 @@ -mod mock_analytics; -#[cfg(feature = "analytics")] -mod segment_analytics; - -use std::fs; -use std::path::{Path, PathBuf}; -use std::str::FromStr; - -use actix_web::HttpRequest; -use meilisearch_types::InstanceUid; -pub use mock_analytics::MockAnalytics; -use once_cell::sync::Lazy; -use platform_dirs::AppDirs; -use serde_json::Value; - -use crate::routes::indexes::documents::{DocumentEditionByFunction, UpdateDocumentsQuery}; - -// if the analytics feature is disabled -// the `SegmentAnalytics` point to the mock instead of the real analytics -#[cfg(not(feature = "analytics"))] -pub type SegmentAnalytics = mock_analytics::MockAnalytics; -#[cfg(not(feature = "analytics"))] -pub type SearchAggregator = mock_analytics::SearchAggregator; -#[cfg(not(feature = "analytics"))] -pub type SimilarAggregator = mock_analytics::SimilarAggregator; -#[cfg(not(feature = "analytics"))] -pub type MultiSearchAggregator = mock_analytics::MultiSearchAggregator; -#[cfg(not(feature = "analytics"))] -pub type FacetSearchAggregator = mock_analytics::FacetSearchAggregator; - -// if the feature analytics is enabled we use the real analytics -#[cfg(feature = "analytics")] -pub type SegmentAnalytics = segment_analytics::SegmentAnalytics; -#[cfg(feature = "analytics")] -pub type SearchAggregator = segment_analytics::SearchAggregator; -#[cfg(feature = "analytics")] -pub type SimilarAggregator = segment_analytics::SimilarAggregator; -#[cfg(feature = "analytics")] -pub type MultiSearchAggregator = segment_analytics::MultiSearchAggregator; -#[cfg(feature = "analytics")] -pub type FacetSearchAggregator = segment_analytics::FacetSearchAggregator; - -/// The Meilisearch config dir: -/// `~/.config/Meilisearch` on *NIX or *BSD. -/// `~/Library/ApplicationSupport` on macOS. -/// `%APPDATA` (= `C:\Users%USERNAME%\AppData\Roaming`) on windows. -static MEILISEARCH_CONFIG_PATH: Lazy> = - Lazy::new(|| AppDirs::new(Some("Meilisearch"), false).map(|appdir| appdir.config_dir)); - -fn config_user_id_path(db_path: &Path) -> Option { - db_path - .canonicalize() - .ok() - .map(|path| path.join("instance-uid").display().to_string().replace('/', "-")) - .zip(MEILISEARCH_CONFIG_PATH.as_ref()) - .map(|(filename, config_path)| config_path.join(filename.trim_start_matches('-'))) -} - -/// Look for the instance-uid in the `data.ms` or in `~/.config/Meilisearch/path-to-db-instance-uid` -fn find_user_id(db_path: &Path) -> Option { - fs::read_to_string(db_path.join("instance-uid")) - .ok() - .or_else(|| fs::read_to_string(config_user_id_path(db_path)?).ok()) - .and_then(|uid| InstanceUid::from_str(&uid).ok()) -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum DocumentDeletionKind { - PerDocumentId, - ClearAll, - PerBatch, - PerFilter, -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum DocumentFetchKind { - PerDocumentId { retrieve_vectors: bool }, - Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool }, -} - -pub trait Analytics: Sync + Send { - fn instance_uid(&self) -> Option<&InstanceUid>; - - /// The method used to publish most analytics that do not need to be batched every hours - fn publish(&self, event_name: String, send: Value, request: Option<&HttpRequest>); - - /// This method should be called to aggregate a get search - fn get_search(&self, aggregate: SearchAggregator); - - /// This method should be called to aggregate a post search - fn post_search(&self, aggregate: SearchAggregator); - - /// This method should be called to aggregate a get similar request - fn get_similar(&self, aggregate: SimilarAggregator); - - /// This method should be called to aggregate a post similar request - fn post_similar(&self, aggregate: SimilarAggregator); - - /// This method should be called to aggregate a post array of searches - fn post_multi_search(&self, aggregate: MultiSearchAggregator); - - /// This method should be called to aggregate post facet values searches - fn post_facet_search(&self, aggregate: FacetSearchAggregator); - - // this method should be called to aggregate an add documents request - fn add_documents( - &self, - documents_query: &UpdateDocumentsQuery, - index_creation: bool, - request: &HttpRequest, - ); - - // this method should be called to aggregate a fetch documents request - fn get_fetch_documents(&self, documents_query: &DocumentFetchKind, request: &HttpRequest); - - // this method should be called to aggregate a fetch documents request - fn post_fetch_documents(&self, documents_query: &DocumentFetchKind, request: &HttpRequest); - - // this method should be called to aggregate a add documents request - fn delete_documents(&self, kind: DocumentDeletionKind, request: &HttpRequest); - - // this method should be called to batch an update documents request - fn update_documents( - &self, - documents_query: &UpdateDocumentsQuery, - index_creation: bool, - request: &HttpRequest, - ); - - // this method should be called to batch an update documents by function request - fn update_documents_by_function( - &self, - documents_query: &DocumentEditionByFunction, - index_creation: bool, - request: &HttpRequest, - ); -} diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs deleted file mode 100644 index f8d6a0fdc..000000000 --- a/meilisearch/src/analytics/segment_analytics.rs +++ /dev/null @@ -1,1994 +0,0 @@ -use std::collections::{BTreeSet, BinaryHeap, HashMap, HashSet}; -use std::fs; -use std::mem::take; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use actix_web::http::header::{CONTENT_TYPE, USER_AGENT}; -use actix_web::HttpRequest; -use byte_unit::Byte; -use index_scheduler::IndexScheduler; -use meilisearch_auth::{AuthController, AuthFilter}; -use meilisearch_types::locales::Locale; -use meilisearch_types::InstanceUid; -use once_cell::sync::Lazy; -use regex::Regex; -use segment::message::{Identify, Track, User}; -use segment::{AutoBatcher, Batcher, HttpClient}; -use serde::Serialize; -use serde_json::{json, Value}; -use sysinfo::{Disks, System}; -use time::OffsetDateTime; -use tokio::select; -use tokio::sync::mpsc::{self, Receiver, Sender}; -use uuid::Uuid; - -use super::{ - config_user_id_path, DocumentDeletionKind, DocumentFetchKind, MEILISEARCH_CONFIG_PATH, -}; -use crate::analytics::Analytics; -use crate::option::{ - default_http_addr, IndexerOpts, LogMode, MaxMemory, MaxThreads, ScheduleSnapshot, -}; -use crate::routes::indexes::documents::{DocumentEditionByFunction, UpdateDocumentsQuery}; -use crate::routes::indexes::facet_search::FacetSearchQuery; -use crate::routes::{create_all_stats, Stats}; -use crate::search::{ - FacetSearchResult, FederatedSearch, MatchingStrategy, SearchQuery, SearchQueryWithIndex, - SearchResult, SimilarQuery, SimilarResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, - DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, - DEFAULT_SEMANTIC_RATIO, -}; -use crate::Opt; - -const ANALYTICS_HEADER: &str = "X-Meilisearch-Client"; - -/// Write the instance-uid in the `data.ms` and in `~/.config/MeiliSearch/path-to-db-instance-uid`. Ignore the errors. -fn write_user_id(db_path: &Path, user_id: &InstanceUid) { - let _ = fs::write(db_path.join("instance-uid"), user_id.to_string()); - if let Some((meilisearch_config_path, user_id_path)) = - MEILISEARCH_CONFIG_PATH.as_ref().zip(config_user_id_path(db_path)) - { - let _ = fs::create_dir_all(meilisearch_config_path); - let _ = fs::write(user_id_path, user_id.to_string()); - } -} - -const SEGMENT_API_KEY: &str = "P3FWhhEsJiEDCuEHpmcN9DHcK4hVfBvb"; - -pub fn extract_user_agents(request: &HttpRequest) -> Vec { - request - .headers() - .get(ANALYTICS_HEADER) - .or_else(|| request.headers().get(USER_AGENT)) - .and_then(|header| header.to_str().ok()) - .unwrap_or("unknown") - .split(';') - .map(str::trim) - .map(ToString::to_string) - .collect() -} - -pub enum AnalyticsMsg { - BatchMessage(Track), - AggregateGetSearch(SearchAggregator), - AggregatePostSearch(SearchAggregator), - AggregateGetSimilar(SimilarAggregator), - AggregatePostSimilar(SimilarAggregator), - AggregatePostMultiSearch(MultiSearchAggregator), - AggregatePostFacetSearch(FacetSearchAggregator), - AggregateAddDocuments(DocumentsAggregator), - AggregateDeleteDocuments(DocumentsDeletionAggregator), - AggregateUpdateDocuments(DocumentsAggregator), - AggregateEditDocumentsByFunction(EditDocumentsByFunctionAggregator), - AggregateGetFetchDocuments(DocumentsFetchAggregator), - AggregatePostFetchDocuments(DocumentsFetchAggregator), -} - -pub struct SegmentAnalytics { - instance_uid: InstanceUid, - sender: Sender, - user: User, -} - -impl SegmentAnalytics { - #[allow(clippy::new_ret_no_self)] - pub async fn new( - opt: &Opt, - index_scheduler: Arc, - auth_controller: Arc, - ) -> Arc { - let instance_uid = super::find_user_id(&opt.db_path); - let first_time_run = instance_uid.is_none(); - let instance_uid = instance_uid.unwrap_or_else(Uuid::new_v4); - write_user_id(&opt.db_path, &instance_uid); - - let client = reqwest::Client::builder().connect_timeout(Duration::from_secs(10)).build(); - - // if reqwest throws an error we won't be able to send analytics - if client.is_err() { - return super::MockAnalytics::new(opt); - } - - let client = - HttpClient::new(client.unwrap(), "https://telemetry.meilisearch.com".to_string()); - let user = User::UserId { user_id: instance_uid.to_string() }; - let mut batcher = AutoBatcher::new(client, Batcher::new(None), SEGMENT_API_KEY.to_string()); - - // If Meilisearch is Launched for the first time: - // 1. Send an event Launched associated to the user `total_launch`. - // 2. Batch an event Launched with the real instance-id and send it in one hour. - if first_time_run { - let _ = batcher - .push(Track { - user: User::UserId { user_id: "total_launch".to_string() }, - event: "Launched".to_string(), - ..Default::default() - }) - .await; - let _ = batcher.flush().await; - let _ = batcher - .push(Track { - user: user.clone(), - event: "Launched".to_string(), - ..Default::default() - }) - .await; - } - - let (sender, inbox) = mpsc::channel(100); // How many analytics can we bufferize - - let segment = Box::new(Segment { - inbox, - user: user.clone(), - opt: opt.clone(), - batcher, - post_search_aggregator: SearchAggregator::default(), - post_multi_search_aggregator: MultiSearchAggregator::default(), - post_facet_search_aggregator: FacetSearchAggregator::default(), - get_search_aggregator: SearchAggregator::default(), - add_documents_aggregator: DocumentsAggregator::default(), - delete_documents_aggregator: DocumentsDeletionAggregator::default(), - update_documents_aggregator: DocumentsAggregator::default(), - edit_documents_by_function_aggregator: EditDocumentsByFunctionAggregator::default(), - get_fetch_documents_aggregator: DocumentsFetchAggregator::default(), - post_fetch_documents_aggregator: DocumentsFetchAggregator::default(), - get_similar_aggregator: SimilarAggregator::default(), - post_similar_aggregator: SimilarAggregator::default(), - }); - tokio::spawn(segment.run(index_scheduler.clone(), auth_controller.clone())); - - let this = Self { instance_uid, sender, user: user.clone() }; - - Arc::new(this) - } -} - -impl super::Analytics for SegmentAnalytics { - fn instance_uid(&self) -> Option<&InstanceUid> { - Some(&self.instance_uid) - } - - fn publish(&self, event_name: String, mut send: Value, request: Option<&HttpRequest>) { - let user_agent = request.map(extract_user_agents); - - send["user-agent"] = json!(user_agent); - let event = Track { - user: self.user.clone(), - event: event_name.clone(), - properties: send, - ..Default::default() - }; - let _ = self.sender.try_send(AnalyticsMsg::BatchMessage(event)); - } - - fn get_search(&self, aggregate: SearchAggregator) { - let _ = self.sender.try_send(AnalyticsMsg::AggregateGetSearch(aggregate)); - } - - fn post_search(&self, aggregate: SearchAggregator) { - let _ = self.sender.try_send(AnalyticsMsg::AggregatePostSearch(aggregate)); - } - - fn get_similar(&self, aggregate: SimilarAggregator) { - let _ = self.sender.try_send(AnalyticsMsg::AggregateGetSimilar(aggregate)); - } - - fn post_similar(&self, aggregate: SimilarAggregator) { - let _ = self.sender.try_send(AnalyticsMsg::AggregatePostSimilar(aggregate)); - } - - fn post_facet_search(&self, aggregate: FacetSearchAggregator) { - let _ = self.sender.try_send(AnalyticsMsg::AggregatePostFacetSearch(aggregate)); - } - - fn post_multi_search(&self, aggregate: MultiSearchAggregator) { - let _ = self.sender.try_send(AnalyticsMsg::AggregatePostMultiSearch(aggregate)); - } - - fn add_documents( - &self, - documents_query: &UpdateDocumentsQuery, - index_creation: bool, - request: &HttpRequest, - ) { - let aggregate = DocumentsAggregator::from_query(documents_query, index_creation, request); - let _ = self.sender.try_send(AnalyticsMsg::AggregateAddDocuments(aggregate)); - } - - fn delete_documents(&self, kind: DocumentDeletionKind, request: &HttpRequest) { - let aggregate = DocumentsDeletionAggregator::from_query(kind, request); - let _ = self.sender.try_send(AnalyticsMsg::AggregateDeleteDocuments(aggregate)); - } - - fn update_documents( - &self, - documents_query: &UpdateDocumentsQuery, - index_creation: bool, - request: &HttpRequest, - ) { - let aggregate = DocumentsAggregator::from_query(documents_query, index_creation, request); - let _ = self.sender.try_send(AnalyticsMsg::AggregateUpdateDocuments(aggregate)); - } - - fn update_documents_by_function( - &self, - documents_query: &DocumentEditionByFunction, - index_creation: bool, - request: &HttpRequest, - ) { - let aggregate = - EditDocumentsByFunctionAggregator::from_query(documents_query, index_creation, request); - let _ = self.sender.try_send(AnalyticsMsg::AggregateEditDocumentsByFunction(aggregate)); - } - - fn get_fetch_documents(&self, documents_query: &DocumentFetchKind, request: &HttpRequest) { - let aggregate = DocumentsFetchAggregator::from_query(documents_query, request); - let _ = self.sender.try_send(AnalyticsMsg::AggregateGetFetchDocuments(aggregate)); - } - - fn post_fetch_documents(&self, documents_query: &DocumentFetchKind, request: &HttpRequest) { - let aggregate = DocumentsFetchAggregator::from_query(documents_query, request); - let _ = self.sender.try_send(AnalyticsMsg::AggregatePostFetchDocuments(aggregate)); - } -} - -/// This structure represent the `infos` field we send in the analytics. -/// It's quite close to the `Opt` structure except all sensitive informations -/// have been simplified to a boolean. -/// It's send as-is in amplitude thus you should never update a name of the -/// struct without the approval of the PM. -#[derive(Debug, Clone, Serialize)] -struct Infos { - env: String, - experimental_contains_filter: bool, - experimental_enable_metrics: bool, - experimental_search_queue_size: usize, - experimental_logs_mode: LogMode, - experimental_replication_parameters: bool, - experimental_enable_logs_route: bool, - experimental_reduce_indexing_memory_usage: bool, - experimental_max_number_of_batched_tasks: usize, - gpu_enabled: bool, - db_path: bool, - import_dump: bool, - dump_dir: bool, - ignore_missing_dump: bool, - ignore_dump_if_db_exists: bool, - import_snapshot: bool, - schedule_snapshot: Option, - snapshot_dir: bool, - ignore_missing_snapshot: bool, - ignore_snapshot_if_db_exists: bool, - http_addr: bool, - http_payload_size_limit: Byte, - task_queue_webhook: bool, - task_webhook_authorization_header: bool, - log_level: String, - max_indexing_memory: MaxMemory, - max_indexing_threads: MaxThreads, - with_configuration_file: bool, - ssl_auth_path: bool, - ssl_cert_path: bool, - ssl_key_path: bool, - ssl_ocsp_path: bool, - ssl_require_auth: bool, - ssl_resumption: bool, - ssl_tickets: bool, -} - -impl From for Infos { - fn from(options: Opt) -> Self { - // We wants to decompose this whole struct by hand to be sure we don't forget - // to add analytics when we add a field in the Opt. - // Thus we must not insert `..` at the end. - let Opt { - db_path, - experimental_contains_filter, - experimental_enable_metrics, - experimental_search_queue_size, - experimental_logs_mode, - experimental_replication_parameters, - experimental_enable_logs_route, - experimental_reduce_indexing_memory_usage, - experimental_max_number_of_batched_tasks, - http_addr, - master_key: _, - env, - task_webhook_url, - task_webhook_authorization_header, - max_index_size: _, - max_task_db_size: _, - http_payload_size_limit, - ssl_cert_path, - ssl_key_path, - ssl_auth_path, - ssl_ocsp_path, - ssl_require_auth, - ssl_resumption, - ssl_tickets, - import_snapshot, - ignore_missing_snapshot, - ignore_snapshot_if_db_exists, - snapshot_dir, - schedule_snapshot, - import_dump, - ignore_missing_dump, - ignore_dump_if_db_exists, - dump_dir, - log_level, - indexer_options, - config_file_path, - #[cfg(feature = "analytics")] - no_analytics: _, - } = options; - - let schedule_snapshot = match schedule_snapshot { - ScheduleSnapshot::Disabled => None, - ScheduleSnapshot::Enabled(interval) => Some(interval), - }; - - let IndexerOpts { max_indexing_memory, max_indexing_threads, skip_index_budget: _ } = - indexer_options; - - // We're going to override every sensible information. - // We consider information sensible if it contains a path, an address, or a key. - Self { - env, - experimental_contains_filter, - experimental_enable_metrics, - experimental_search_queue_size, - experimental_logs_mode, - experimental_replication_parameters, - experimental_enable_logs_route, - experimental_reduce_indexing_memory_usage, - gpu_enabled: meilisearch_types::milli::vector::is_cuda_enabled(), - db_path: db_path != PathBuf::from("./data.ms"), - import_dump: import_dump.is_some(), - dump_dir: dump_dir != PathBuf::from("dumps/"), - ignore_missing_dump, - ignore_dump_if_db_exists, - import_snapshot: import_snapshot.is_some(), - schedule_snapshot, - snapshot_dir: snapshot_dir != PathBuf::from("snapshots/"), - ignore_missing_snapshot, - ignore_snapshot_if_db_exists, - http_addr: http_addr != default_http_addr(), - http_payload_size_limit, - experimental_max_number_of_batched_tasks, - task_queue_webhook: task_webhook_url.is_some(), - task_webhook_authorization_header: task_webhook_authorization_header.is_some(), - log_level: log_level.to_string(), - max_indexing_memory, - max_indexing_threads, - with_configuration_file: config_file_path.is_some(), - ssl_auth_path: ssl_auth_path.is_some(), - ssl_cert_path: ssl_cert_path.is_some(), - ssl_key_path: ssl_key_path.is_some(), - ssl_ocsp_path: ssl_ocsp_path.is_some(), - ssl_require_auth, - ssl_resumption, - ssl_tickets, - } - } -} - -pub struct Segment { - inbox: Receiver, - user: User, - opt: Opt, - batcher: AutoBatcher, - get_search_aggregator: SearchAggregator, - post_search_aggregator: SearchAggregator, - post_multi_search_aggregator: MultiSearchAggregator, - post_facet_search_aggregator: FacetSearchAggregator, - add_documents_aggregator: DocumentsAggregator, - delete_documents_aggregator: DocumentsDeletionAggregator, - update_documents_aggregator: DocumentsAggregator, - edit_documents_by_function_aggregator: EditDocumentsByFunctionAggregator, - get_fetch_documents_aggregator: DocumentsFetchAggregator, - post_fetch_documents_aggregator: DocumentsFetchAggregator, - get_similar_aggregator: SimilarAggregator, - post_similar_aggregator: SimilarAggregator, -} - -impl Segment { - fn compute_traits(opt: &Opt, stats: Stats) -> Value { - static FIRST_START_TIMESTAMP: Lazy = Lazy::new(Instant::now); - static SYSTEM: Lazy = Lazy::new(|| { - let disks = Disks::new_with_refreshed_list(); - let mut sys = System::new_all(); - sys.refresh_all(); - let kernel_version = System::kernel_version() - .and_then(|k| k.split_once('-').map(|(k, _)| k.to_string())); - json!({ - "distribution": System::name(), - "kernel_version": kernel_version, - "cores": sys.cpus().len(), - "ram_size": sys.total_memory(), - "disk_size": disks.iter().map(|disk| disk.total_space()).max(), - "server_provider": std::env::var("MEILI_SERVER_PROVIDER").ok(), - }) - }); - let number_of_documents = - stats.indexes.values().map(|index| index.number_of_documents).collect::>(); - - json!({ - "start_since_days": FIRST_START_TIMESTAMP.elapsed().as_secs() / (60 * 60 * 24), // one day - "system": *SYSTEM, - "stats": { - "database_size": stats.database_size, - "indexes_number": stats.indexes.len(), - "documents_number": number_of_documents, - }, - "infos": Infos::from(opt.clone()), - }) - } - - async fn run( - mut self, - index_scheduler: Arc, - auth_controller: Arc, - ) { - const INTERVAL: Duration = Duration::from_secs(60 * 60); // one hour - // The first batch must be sent after one hour. - let mut interval = - tokio::time::interval_at(tokio::time::Instant::now() + INTERVAL, INTERVAL); - - loop { - select! { - _ = interval.tick() => { - self.tick(index_scheduler.clone(), auth_controller.clone()).await; - }, - msg = self.inbox.recv() => { - match msg { - Some(AnalyticsMsg::BatchMessage(msg)) => drop(self.batcher.push(msg).await), - Some(AnalyticsMsg::AggregateGetSearch(agreg)) => self.get_search_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregatePostSearch(agreg)) => self.post_search_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregatePostMultiSearch(agreg)) => self.post_multi_search_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregatePostFacetSearch(agreg)) => self.post_facet_search_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregateAddDocuments(agreg)) => self.add_documents_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregateDeleteDocuments(agreg)) => self.delete_documents_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregateUpdateDocuments(agreg)) => self.update_documents_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregateEditDocumentsByFunction(agreg)) => self.edit_documents_by_function_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregateGetFetchDocuments(agreg)) => self.get_fetch_documents_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregatePostFetchDocuments(agreg)) => self.post_fetch_documents_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregateGetSimilar(agreg)) => self.get_similar_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregatePostSimilar(agreg)) => self.post_similar_aggregator.aggregate(agreg), - None => (), - } - } - } - } - } - - async fn tick( - &mut self, - index_scheduler: Arc, - auth_controller: Arc, - ) { - if let Ok(stats) = - create_all_stats(index_scheduler.into(), auth_controller.into(), &AuthFilter::default()) - { - // Replace the version number with the prototype name if any. - let version = if let Some(prototype) = build_info::DescribeResult::from_build() - .and_then(|describe| describe.as_prototype()) - { - prototype - } else { - env!("CARGO_PKG_VERSION") - }; - - let _ = self - .batcher - .push(Identify { - context: Some(json!({ - "app": { - "version": version.to_string(), - }, - })), - user: self.user.clone(), - traits: Self::compute_traits(&self.opt, stats), - ..Default::default() - }) - .await; - } - - let Segment { - inbox: _, - opt: _, - batcher: _, - user, - get_search_aggregator, - post_search_aggregator, - post_multi_search_aggregator, - post_facet_search_aggregator, - add_documents_aggregator, - delete_documents_aggregator, - update_documents_aggregator, - edit_documents_by_function_aggregator, - get_fetch_documents_aggregator, - post_fetch_documents_aggregator, - get_similar_aggregator, - post_similar_aggregator, - } = self; - - if let Some(get_search) = - take(get_search_aggregator).into_event(user, "Documents Searched GET") - { - let _ = self.batcher.push(get_search).await; - } - if let Some(post_search) = - take(post_search_aggregator).into_event(user, "Documents Searched POST") - { - let _ = self.batcher.push(post_search).await; - } - if let Some(post_multi_search) = take(post_multi_search_aggregator) - .into_event(user, "Documents Searched by Multi-Search POST") - { - let _ = self.batcher.push(post_multi_search).await; - } - if let Some(post_facet_search) = - take(post_facet_search_aggregator).into_event(user, "Facet Searched POST") - { - let _ = self.batcher.push(post_facet_search).await; - } - if let Some(add_documents) = - take(add_documents_aggregator).into_event(user, "Documents Added") - { - let _ = self.batcher.push(add_documents).await; - } - if let Some(delete_documents) = - take(delete_documents_aggregator).into_event(user, "Documents Deleted") - { - let _ = self.batcher.push(delete_documents).await; - } - if let Some(update_documents) = - take(update_documents_aggregator).into_event(user, "Documents Updated") - { - let _ = self.batcher.push(update_documents).await; - } - if let Some(edit_documents_by_function) = take(edit_documents_by_function_aggregator) - .into_event(user, "Documents Edited By Function") - { - let _ = self.batcher.push(edit_documents_by_function).await; - } - if let Some(get_fetch_documents) = - take(get_fetch_documents_aggregator).into_event(user, "Documents Fetched GET") - { - let _ = self.batcher.push(get_fetch_documents).await; - } - if let Some(post_fetch_documents) = - take(post_fetch_documents_aggregator).into_event(user, "Documents Fetched POST") - { - let _ = self.batcher.push(post_fetch_documents).await; - } - - if let Some(get_similar_documents) = - take(get_similar_aggregator).into_event(user, "Similar GET") - { - let _ = self.batcher.push(get_similar_documents).await; - } - - if let Some(post_similar_documents) = - take(post_similar_aggregator).into_event(user, "Similar POST") - { - let _ = self.batcher.push(post_similar_documents).await; - } - let _ = self.batcher.flush().await; - } -} - -#[derive(Default)] -pub struct SearchAggregator { - timestamp: Option, - - // context - user_agents: HashSet, - - // requests - total_received: usize, - total_succeeded: usize, - total_degraded: usize, - total_used_negative_operator: usize, - time_spent: BinaryHeap, - - // sort - sort_with_geo_point: bool, - // every time a request has a filter, this field must be incremented by the number of terms it contains - sort_sum_of_criteria_terms: usize, - // every time a request has a filter, this field must be incremented by one - sort_total_number_of_criteria: usize, - - // distinct - distinct: bool, - - // filter - filter_with_geo_radius: bool, - filter_with_geo_bounding_box: bool, - // every time a request has a filter, this field must be incremented by the number of terms it contains - filter_sum_of_criteria_terms: usize, - // every time a request has a filter, this field must be incremented by one - filter_total_number_of_criteria: usize, - used_syntax: HashMap, - - // attributes_to_search_on - // every time a search is done using attributes_to_search_on - attributes_to_search_on_total_number_of_uses: usize, - - // q - // The maximum number of terms in a q request - max_terms_number: usize, - - // vector - // The maximum number of floats in a vector request - max_vector_size: usize, - // Whether the semantic ratio passed to a hybrid search equals the default ratio. - semantic_ratio: bool, - hybrid: bool, - retrieve_vectors: bool, - - // every time a search is done, we increment the counter linked to the used settings - matching_strategy: HashMap, - - // List of the unique Locales passed as parameter - locales: BTreeSet, - - // pagination - max_limit: usize, - max_offset: usize, - finite_pagination: usize, - - // formatting - max_attributes_to_retrieve: usize, - max_attributes_to_highlight: usize, - highlight_pre_tag: bool, - highlight_post_tag: bool, - max_attributes_to_crop: usize, - crop_marker: bool, - show_matches_position: bool, - crop_length: bool, - - // facets - facets_sum_of_terms: usize, - facets_total_number_of_facets: usize, - - // scoring - show_ranking_score: bool, - show_ranking_score_details: bool, - ranking_score_threshold: bool, -} - -impl SearchAggregator { - #[allow(clippy::field_reassign_with_default)] - pub fn from_query(query: &SearchQuery, request: &HttpRequest) -> Self { - let SearchQuery { - q, - vector, - offset, - limit, - page, - hits_per_page, - attributes_to_retrieve: _, - retrieve_vectors, - attributes_to_crop: _, - crop_length, - attributes_to_highlight: _, - show_matches_position, - show_ranking_score, - show_ranking_score_details, - filter, - sort, - distinct, - facets: _, - highlight_pre_tag, - highlight_post_tag, - crop_marker, - matching_strategy, - attributes_to_search_on, - hybrid, - ranking_score_threshold, - locales, - } = query; - - let mut ret = Self::default(); - ret.timestamp = Some(OffsetDateTime::now_utc()); - - ret.total_received = 1; - ret.user_agents = extract_user_agents(request).into_iter().collect(); - - if let Some(ref sort) = sort { - ret.sort_total_number_of_criteria = 1; - ret.sort_with_geo_point = sort.iter().any(|s| s.contains("_geoPoint(")); - ret.sort_sum_of_criteria_terms = sort.len(); - } - - ret.distinct = distinct.is_some(); - - if let Some(ref filter) = filter { - static RE: Lazy = Lazy::new(|| Regex::new("AND | OR").unwrap()); - ret.filter_total_number_of_criteria = 1; - - let syntax = match filter { - Value::String(_) => "string".to_string(), - Value::Array(values) => { - if values.iter().map(|v| v.to_string()).any(|s| RE.is_match(&s)) { - "mixed".to_string() - } else { - "array".to_string() - } - } - _ => "none".to_string(), - }; - // convert the string to a HashMap - ret.used_syntax.insert(syntax, 1); - - let stringified_filters = filter.to_string(); - ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius("); - ret.filter_with_geo_bounding_box = stringified_filters.contains("_geoBoundingBox("); - ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count(); - } - - // attributes_to_search_on - if attributes_to_search_on.is_some() { - ret.attributes_to_search_on_total_number_of_uses = 1; - } - - if let Some(ref q) = q { - ret.max_terms_number = q.split_whitespace().count(); - } - - if let Some(ref vector) = vector { - ret.max_vector_size = vector.len(); - } - ret.retrieve_vectors |= retrieve_vectors; - - if query.is_finite_pagination() { - let limit = hits_per_page.unwrap_or_else(DEFAULT_SEARCH_LIMIT); - ret.max_limit = limit; - ret.max_offset = page.unwrap_or(1).saturating_sub(1) * limit; - ret.finite_pagination = 1; - } else { - ret.max_limit = *limit; - ret.max_offset = *offset; - ret.finite_pagination = 0; - } - - ret.matching_strategy.insert(format!("{:?}", matching_strategy), 1); - - if let Some(locales) = locales { - ret.locales = locales.iter().copied().collect(); - } - - ret.highlight_pre_tag = *highlight_pre_tag != DEFAULT_HIGHLIGHT_PRE_TAG(); - ret.highlight_post_tag = *highlight_post_tag != DEFAULT_HIGHLIGHT_POST_TAG(); - ret.crop_marker = *crop_marker != DEFAULT_CROP_MARKER(); - ret.crop_length = *crop_length != DEFAULT_CROP_LENGTH(); - ret.show_matches_position = *show_matches_position; - - ret.show_ranking_score = *show_ranking_score; - ret.show_ranking_score_details = *show_ranking_score_details; - ret.ranking_score_threshold = ranking_score_threshold.is_some(); - - if let Some(hybrid) = hybrid { - ret.semantic_ratio = hybrid.semantic_ratio != DEFAULT_SEMANTIC_RATIO(); - ret.hybrid = true; - } - - ret - } - - pub fn succeed(&mut self, result: &SearchResult) { - let SearchResult { - hits: _, - query: _, - processing_time_ms, - hits_info: _, - semantic_hit_count: _, - facet_distribution: _, - facet_stats: _, - degraded, - used_negative_operator, - } = result; - - self.total_succeeded = self.total_succeeded.saturating_add(1); - if *degraded { - self.total_degraded = self.total_degraded.saturating_add(1); - } - if *used_negative_operator { - self.total_used_negative_operator = self.total_used_negative_operator.saturating_add(1); - } - self.time_spent.push(*processing_time_ms as usize); - } - - /// Aggregate one [SearchAggregator] into another. - pub fn aggregate(&mut self, mut other: Self) { - let Self { - timestamp, - user_agents, - total_received, - total_succeeded, - ref mut time_spent, - sort_with_geo_point, - sort_sum_of_criteria_terms, - sort_total_number_of_criteria, - distinct, - filter_with_geo_radius, - filter_with_geo_bounding_box, - filter_sum_of_criteria_terms, - filter_total_number_of_criteria, - used_syntax, - attributes_to_search_on_total_number_of_uses, - max_terms_number, - max_vector_size, - retrieve_vectors, - matching_strategy, - max_limit, - max_offset, - finite_pagination, - max_attributes_to_retrieve, - max_attributes_to_highlight, - highlight_pre_tag, - highlight_post_tag, - max_attributes_to_crop, - crop_marker, - show_matches_position, - crop_length, - facets_sum_of_terms, - facets_total_number_of_facets, - show_ranking_score, - show_ranking_score_details, - semantic_ratio, - hybrid, - total_degraded, - total_used_negative_operator, - ranking_score_threshold, - ref mut locales, - } = other; - - if self.timestamp.is_none() { - self.timestamp = timestamp; - } - - // context - for user_agent in user_agents.into_iter() { - self.user_agents.insert(user_agent); - } - - // request - self.total_received = self.total_received.saturating_add(total_received); - self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded); - self.total_degraded = self.total_degraded.saturating_add(total_degraded); - self.total_used_negative_operator = - self.total_used_negative_operator.saturating_add(total_used_negative_operator); - self.time_spent.append(time_spent); - - // sort - self.sort_with_geo_point |= sort_with_geo_point; - self.sort_sum_of_criteria_terms = - self.sort_sum_of_criteria_terms.saturating_add(sort_sum_of_criteria_terms); - self.sort_total_number_of_criteria = - self.sort_total_number_of_criteria.saturating_add(sort_total_number_of_criteria); - - // distinct - self.distinct |= distinct; - - // filter - self.filter_with_geo_radius |= filter_with_geo_radius; - self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box; - self.filter_sum_of_criteria_terms = - self.filter_sum_of_criteria_terms.saturating_add(filter_sum_of_criteria_terms); - self.filter_total_number_of_criteria = - self.filter_total_number_of_criteria.saturating_add(filter_total_number_of_criteria); - for (key, value) in used_syntax.into_iter() { - let used_syntax = self.used_syntax.entry(key).or_insert(0); - *used_syntax = used_syntax.saturating_add(value); - } - - // attributes_to_search_on - self.attributes_to_search_on_total_number_of_uses = self - .attributes_to_search_on_total_number_of_uses - .saturating_add(attributes_to_search_on_total_number_of_uses); - - // q - self.max_terms_number = self.max_terms_number.max(max_terms_number); - - // vector - self.max_vector_size = self.max_vector_size.max(max_vector_size); - self.retrieve_vectors |= retrieve_vectors; - self.semantic_ratio |= semantic_ratio; - self.hybrid |= hybrid; - - // pagination - self.max_limit = self.max_limit.max(max_limit); - self.max_offset = self.max_offset.max(max_offset); - self.finite_pagination += finite_pagination; - - // formatting - self.max_attributes_to_retrieve = - self.max_attributes_to_retrieve.max(max_attributes_to_retrieve); - self.max_attributes_to_highlight = - self.max_attributes_to_highlight.max(max_attributes_to_highlight); - self.highlight_pre_tag |= highlight_pre_tag; - self.highlight_post_tag |= highlight_post_tag; - self.max_attributes_to_crop = self.max_attributes_to_crop.max(max_attributes_to_crop); - self.crop_marker |= crop_marker; - self.show_matches_position |= show_matches_position; - self.crop_length |= crop_length; - - // facets - self.facets_sum_of_terms = self.facets_sum_of_terms.saturating_add(facets_sum_of_terms); - self.facets_total_number_of_facets = - self.facets_total_number_of_facets.saturating_add(facets_total_number_of_facets); - - // matching strategy - for (key, value) in matching_strategy.into_iter() { - let matching_strategy = self.matching_strategy.entry(key).or_insert(0); - *matching_strategy = matching_strategy.saturating_add(value); - } - - // scoring - self.show_ranking_score |= show_ranking_score; - self.show_ranking_score_details |= show_ranking_score_details; - self.ranking_score_threshold |= ranking_score_threshold; - - // locales - self.locales.append(locales); - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - let Self { - timestamp, - user_agents, - total_received, - total_succeeded, - time_spent, - sort_with_geo_point, - sort_sum_of_criteria_terms, - sort_total_number_of_criteria, - distinct, - filter_with_geo_radius, - filter_with_geo_bounding_box, - filter_sum_of_criteria_terms, - filter_total_number_of_criteria, - used_syntax, - attributes_to_search_on_total_number_of_uses, - max_terms_number, - max_vector_size, - retrieve_vectors, - matching_strategy, - max_limit, - max_offset, - finite_pagination, - max_attributes_to_retrieve, - max_attributes_to_highlight, - highlight_pre_tag, - highlight_post_tag, - max_attributes_to_crop, - crop_marker, - show_matches_position, - crop_length, - facets_sum_of_terms, - facets_total_number_of_facets, - show_ranking_score, - show_ranking_score_details, - semantic_ratio, - hybrid, - total_degraded, - total_used_negative_operator, - ranking_score_threshold, - locales, - } = self; - - if total_received == 0 { - None - } else { - // we get all the values in a sorted manner - let time_spent = time_spent.into_sorted_vec(); - // the index of the 99th percentage of value - let percentile_99th = time_spent.len() * 99 / 100; - // We are only interested by the slowest value of the 99th fastest results - let time_spent = time_spent.get(percentile_99th); - - let properties = json!({ - "user-agent": user_agents, - "requests": { - "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), - "total_succeeded": total_succeeded, - "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics - "total_received": total_received, - "total_degraded": total_degraded, - "total_used_negative_operator": total_used_negative_operator, - }, - "sort": { - "with_geoPoint": sort_with_geo_point, - "avg_criteria_number": format!("{:.2}", sort_sum_of_criteria_terms as f64 / sort_total_number_of_criteria as f64), - }, - "distinct": distinct, - "filter": { - "with_geoRadius": filter_with_geo_radius, - "with_geoBoundingBox": filter_with_geo_bounding_box, - "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64), - "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), - }, - "attributes_to_search_on": { - "total_number_of_uses": attributes_to_search_on_total_number_of_uses, - }, - "q": { - "max_terms_number": max_terms_number, - }, - "vector": { - "max_vector_size": max_vector_size, - "retrieve_vectors": retrieve_vectors, - }, - "hybrid": { - "enabled": hybrid, - "semantic_ratio": semantic_ratio, - }, - "pagination": { - "max_limit": max_limit, - "max_offset": max_offset, - "most_used_navigation": if finite_pagination > (total_received / 2) { "exhaustive" } else { "estimated" }, - }, - "formatting": { - "max_attributes_to_retrieve": max_attributes_to_retrieve, - "max_attributes_to_highlight": max_attributes_to_highlight, - "highlight_pre_tag": highlight_pre_tag, - "highlight_post_tag": highlight_post_tag, - "max_attributes_to_crop": max_attributes_to_crop, - "crop_marker": crop_marker, - "show_matches_position": show_matches_position, - "crop_length": crop_length, - }, - "facets": { - "avg_facets_number": format!("{:.2}", facets_sum_of_terms as f64 / facets_total_number_of_facets as f64), - }, - "matching_strategy": { - "most_used_strategy": matching_strategy.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), - }, - "locales": locales, - "scoring": { - "show_ranking_score": show_ranking_score, - "show_ranking_score_details": show_ranking_score_details, - "ranking_score_threshold": ranking_score_threshold, - }, - }); - - Some(Track { - timestamp, - user: user.clone(), - event: event_name.to_string(), - properties, - ..Default::default() - }) - } - } -} - -#[derive(Default)] -pub struct MultiSearchAggregator { - timestamp: Option, - - // requests - total_received: usize, - total_succeeded: usize, - - // sum of the number of distinct indexes in each single request, use with total_received to compute an avg - total_distinct_index_count: usize, - // number of queries with a single index, use with total_received to compute a proportion - total_single_index: usize, - - // sum of the number of search queries in the requests, use with total_received to compute an average - total_search_count: usize, - - // scoring - show_ranking_score: bool, - show_ranking_score_details: bool, - - // federation - use_federation: bool, - - // context - user_agents: HashSet, -} - -impl MultiSearchAggregator { - pub fn from_federated_search( - federated_search: &FederatedSearch, - request: &HttpRequest, - ) -> Self { - let timestamp = Some(OffsetDateTime::now_utc()); - - let user_agents = extract_user_agents(request).into_iter().collect(); - - let use_federation = federated_search.federation.is_some(); - - let distinct_indexes: HashSet<_> = federated_search - .queries - .iter() - .map(|query| { - let query = &query; - // make sure we get a compilation error if a field gets added to / removed from SearchQueryWithIndex - let SearchQueryWithIndex { - index_uid, - federation_options: _, - q: _, - vector: _, - offset: _, - limit: _, - page: _, - hits_per_page: _, - attributes_to_retrieve: _, - retrieve_vectors: _, - attributes_to_crop: _, - crop_length: _, - attributes_to_highlight: _, - show_ranking_score: _, - show_ranking_score_details: _, - show_matches_position: _, - filter: _, - sort: _, - distinct: _, - facets: _, - highlight_pre_tag: _, - highlight_post_tag: _, - crop_marker: _, - matching_strategy: _, - attributes_to_search_on: _, - hybrid: _, - ranking_score_threshold: _, - locales: _, - } = query; - - index_uid.as_str() - }) - .collect(); - - let show_ranking_score = - federated_search.queries.iter().any(|query| query.show_ranking_score); - let show_ranking_score_details = - federated_search.queries.iter().any(|query| query.show_ranking_score_details); - - Self { - timestamp, - total_received: 1, - total_succeeded: 0, - total_distinct_index_count: distinct_indexes.len(), - total_single_index: if distinct_indexes.len() == 1 { 1 } else { 0 }, - total_search_count: federated_search.queries.len(), - show_ranking_score, - show_ranking_score_details, - user_agents, - use_federation, - } - } - - pub fn succeed(&mut self) { - self.total_succeeded = self.total_succeeded.saturating_add(1); - } - - /// Aggregate one [MultiSearchAggregator] into another. - pub fn aggregate(&mut self, other: Self) { - // write the aggregate in a way that will cause a compilation error if a field is added. - - // get ownership of self, replacing it by a default value. - let this = std::mem::take(self); - - let timestamp = this.timestamp.or(other.timestamp); - let total_received = this.total_received.saturating_add(other.total_received); - let total_succeeded = this.total_succeeded.saturating_add(other.total_succeeded); - let total_distinct_index_count = - this.total_distinct_index_count.saturating_add(other.total_distinct_index_count); - let total_single_index = this.total_single_index.saturating_add(other.total_single_index); - let total_search_count = this.total_search_count.saturating_add(other.total_search_count); - let show_ranking_score = this.show_ranking_score || other.show_ranking_score; - let show_ranking_score_details = - this.show_ranking_score_details || other.show_ranking_score_details; - let mut user_agents = this.user_agents; - let use_federation = this.use_federation || other.use_federation; - - for user_agent in other.user_agents.into_iter() { - user_agents.insert(user_agent); - } - - // need all fields or compile error - let mut aggregated = Self { - timestamp, - total_received, - total_succeeded, - total_distinct_index_count, - total_single_index, - total_search_count, - user_agents, - show_ranking_score, - show_ranking_score_details, - use_federation, - // do not add _ or ..Default::default() here - }; - - // replace the default self with the aggregated value - std::mem::swap(self, &mut aggregated); - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - let Self { - timestamp, - total_received, - total_succeeded, - total_distinct_index_count, - total_single_index, - total_search_count, - user_agents, - show_ranking_score, - show_ranking_score_details, - use_federation, - } = self; - - if total_received == 0 { - None - } else { - let properties = json!({ - "user-agent": user_agents, - "requests": { - "total_succeeded": total_succeeded, - "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics - "total_received": total_received, - }, - "indexes": { - "total_single_index": total_single_index, - "total_distinct_index_count": total_distinct_index_count, - "avg_distinct_index_count": (total_distinct_index_count as f64) / (total_received as f64), // not 0 else returned early - }, - "searches": { - "total_search_count": total_search_count, - "avg_search_count": (total_search_count as f64) / (total_received as f64), - }, - "scoring": { - "show_ranking_score": show_ranking_score, - "show_ranking_score_details": show_ranking_score_details, - }, - "federation": { - "use_federation": use_federation, - } - }); - - Some(Track { - timestamp, - user: user.clone(), - event: event_name.to_string(), - properties, - ..Default::default() - }) - } - } -} - -#[derive(Default)] -pub struct FacetSearchAggregator { - timestamp: Option, - - // context - user_agents: HashSet, - - // requests - total_received: usize, - total_succeeded: usize, - time_spent: BinaryHeap, - - // The set of all facetNames that were used - facet_names: HashSet, - - // As there been any other parameter than the facetName or facetQuery ones? - additional_search_parameters_provided: bool, -} - -impl FacetSearchAggregator { - #[allow(clippy::field_reassign_with_default)] - pub fn from_query(query: &FacetSearchQuery, request: &HttpRequest) -> Self { - let FacetSearchQuery { - facet_query: _, - facet_name, - vector, - q, - filter, - matching_strategy, - attributes_to_search_on, - hybrid, - ranking_score_threshold, - locales, - } = query; - - let mut ret = Self::default(); - ret.timestamp = Some(OffsetDateTime::now_utc()); - - ret.total_received = 1; - ret.user_agents = extract_user_agents(request).into_iter().collect(); - ret.facet_names = Some(facet_name.clone()).into_iter().collect(); - - ret.additional_search_parameters_provided = q.is_some() - || vector.is_some() - || filter.is_some() - || *matching_strategy != MatchingStrategy::default() - || attributes_to_search_on.is_some() - || hybrid.is_some() - || ranking_score_threshold.is_some() - || locales.is_some(); - - ret - } - - pub fn succeed(&mut self, result: &FacetSearchResult) { - let FacetSearchResult { facet_hits: _, facet_query: _, processing_time_ms } = result; - self.total_succeeded = self.total_succeeded.saturating_add(1); - self.time_spent.push(*processing_time_ms as usize); - } - - /// Aggregate one [FacetSearchAggregator] into another. - pub fn aggregate(&mut self, mut other: Self) { - let Self { - timestamp, - user_agents, - total_received, - total_succeeded, - ref mut time_spent, - facet_names, - additional_search_parameters_provided, - } = other; - - if self.timestamp.is_none() { - self.timestamp = timestamp; - } - - // context - for user_agent in user_agents.into_iter() { - self.user_agents.insert(user_agent); - } - - // request - self.total_received = self.total_received.saturating_add(total_received); - self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded); - self.time_spent.append(time_spent); - - // facet_names - for facet_name in facet_names.into_iter() { - self.facet_names.insert(facet_name); - } - - // additional_search_parameters_provided - self.additional_search_parameters_provided |= additional_search_parameters_provided; - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - let Self { - timestamp, - user_agents, - total_received, - total_succeeded, - time_spent, - facet_names, - additional_search_parameters_provided, - } = self; - - if total_received == 0 { - None - } else { - // the index of the 99th percentage of value - let percentile_99th = 0.99 * (total_succeeded as f64 - 1.) + 1.; - // we get all the values in a sorted manner - let time_spent = time_spent.into_sorted_vec(); - // We are only interested by the slowest value of the 99th fastest results - let time_spent = time_spent.get(percentile_99th as usize); - - let properties = json!({ - "user-agent": user_agents, - "requests": { - "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), - "total_succeeded": total_succeeded, - "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics - "total_received": total_received, - }, - "facets": { - "total_distinct_facet_count": facet_names.len(), - "additional_search_parameters_provided": additional_search_parameters_provided, - }, - }); - - Some(Track { - timestamp, - user: user.clone(), - event: event_name.to_string(), - properties, - ..Default::default() - }) - } - } -} - -#[derive(Default)] -pub struct DocumentsAggregator { - timestamp: Option, - - // set to true when at least one request was received - updated: bool, - - // context - user_agents: HashSet, - - content_types: HashSet, - primary_keys: HashSet, - index_creation: bool, -} - -impl DocumentsAggregator { - pub fn from_query( - documents_query: &UpdateDocumentsQuery, - index_creation: bool, - request: &HttpRequest, - ) -> Self { - let UpdateDocumentsQuery { primary_key, csv_delimiter: _ } = documents_query; - - let mut primary_keys = HashSet::new(); - if let Some(primary_key) = primary_key.clone() { - primary_keys.insert(primary_key); - } - - let mut content_types = HashSet::new(); - let content_type = request - .headers() - .get(CONTENT_TYPE) - .and_then(|s| s.to_str().ok()) - .unwrap_or("unknown") - .to_string(); - content_types.insert(content_type); - - Self { - timestamp: Some(OffsetDateTime::now_utc()), - updated: true, - user_agents: extract_user_agents(request).into_iter().collect(), - content_types, - primary_keys, - index_creation, - } - } - - /// Aggregate one [DocumentsAggregator] into another. - pub fn aggregate(&mut self, other: Self) { - let Self { timestamp, user_agents, primary_keys, content_types, index_creation, updated } = - other; - - if self.timestamp.is_none() { - self.timestamp = timestamp; - } - - self.updated |= updated; - // we can't create a union because there is no `into_union` method - for user_agent in user_agents { - self.user_agents.insert(user_agent); - } - for primary_key in primary_keys { - self.primary_keys.insert(primary_key); - } - for content_type in content_types { - self.content_types.insert(content_type); - } - self.index_creation |= index_creation; - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - let Self { timestamp, user_agents, primary_keys, content_types, index_creation, updated } = - self; - - if !updated { - None - } else { - let properties = json!({ - "user-agent": user_agents, - "payload_type": content_types, - "primary_key": primary_keys, - "index_creation": index_creation, - }); - - Some(Track { - timestamp, - user: user.clone(), - event: event_name.to_string(), - properties, - ..Default::default() - }) - } - } -} - -#[derive(Default)] -pub struct EditDocumentsByFunctionAggregator { - timestamp: Option, - - // Set to true if at least one request was filtered - filtered: bool, - // Set to true if at least one request contained a context - with_context: bool, - - // context - user_agents: HashSet, - - index_creation: bool, -} - -impl EditDocumentsByFunctionAggregator { - pub fn from_query( - documents_query: &DocumentEditionByFunction, - index_creation: bool, - request: &HttpRequest, - ) -> Self { - let DocumentEditionByFunction { filter, context, function: _ } = documents_query; - - Self { - timestamp: Some(OffsetDateTime::now_utc()), - user_agents: extract_user_agents(request).into_iter().collect(), - filtered: filter.is_some(), - with_context: context.is_some(), - index_creation, - } - } - - /// Aggregate one [DocumentsAggregator] into another. - pub fn aggregate(&mut self, other: Self) { - let Self { timestamp, user_agents, index_creation, filtered, with_context } = other; - - if self.timestamp.is_none() { - self.timestamp = timestamp; - } - - // we can't create a union because there is no `into_union` method - for user_agent in user_agents { - self.user_agents.insert(user_agent); - } - self.index_creation |= index_creation; - self.filtered |= filtered; - self.with_context |= with_context; - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - let Self { timestamp, user_agents, index_creation, filtered, with_context } = self; - - let properties = json!({ - "user-agent": user_agents, - "filtered": filtered, - "with_context": with_context, - "index_creation": index_creation, - }); - - Some(Track { - timestamp, - user: user.clone(), - event: event_name.to_string(), - properties, - ..Default::default() - }) - } -} - -#[derive(Default, Serialize)] -pub struct DocumentsDeletionAggregator { - #[serde(skip)] - timestamp: Option, - - // context - #[serde(rename = "user-agent")] - user_agents: HashSet, - - #[serde(rename = "requests.total_received")] - total_received: usize, - per_document_id: bool, - clear_all: bool, - per_batch: bool, - per_filter: bool, -} - -impl DocumentsDeletionAggregator { - pub fn from_query(kind: DocumentDeletionKind, request: &HttpRequest) -> Self { - Self { - timestamp: Some(OffsetDateTime::now_utc()), - user_agents: extract_user_agents(request).into_iter().collect(), - total_received: 1, - per_document_id: matches!(kind, DocumentDeletionKind::PerDocumentId), - clear_all: matches!(kind, DocumentDeletionKind::ClearAll), - per_batch: matches!(kind, DocumentDeletionKind::PerBatch), - per_filter: matches!(kind, DocumentDeletionKind::PerFilter), - } - } - - /// Aggregate one [DocumentsAggregator] into another. - pub fn aggregate(&mut self, other: Self) { - let Self { - timestamp, - user_agents, - total_received, - per_document_id, - clear_all, - per_batch, - per_filter, - } = other; - - if self.timestamp.is_none() { - self.timestamp = timestamp; - } - - // we can't create a union because there is no `into_union` method - for user_agent in user_agents { - self.user_agents.insert(user_agent); - } - self.total_received = self.total_received.saturating_add(total_received); - self.per_document_id |= per_document_id; - self.clear_all |= clear_all; - self.per_batch |= per_batch; - self.per_filter |= per_filter; - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - // if we had no timestamp it means we never encountered any events and - // thus we don't need to send this event. - let timestamp = self.timestamp?; - - Some(Track { - timestamp: Some(timestamp), - user: user.clone(), - event: event_name.to_string(), - properties: serde_json::to_value(self).ok()?, - ..Default::default() - }) - } -} - -#[derive(Default, Serialize)] -pub struct DocumentsFetchAggregator { - #[serde(skip)] - timestamp: Option, - - // context - #[serde(rename = "user-agent")] - user_agents: HashSet, - - #[serde(rename = "requests.total_received")] - total_received: usize, - - // a call on ../documents/:doc_id - per_document_id: bool, - // if a filter was used - per_filter: bool, - - #[serde(rename = "vector.retrieve_vectors")] - retrieve_vectors: bool, - - // pagination - #[serde(rename = "pagination.max_limit")] - max_limit: usize, - #[serde(rename = "pagination.max_offset")] - max_offset: usize, -} - -impl DocumentsFetchAggregator { - pub fn from_query(query: &DocumentFetchKind, request: &HttpRequest) -> Self { - let (limit, offset, retrieve_vectors) = match query { - DocumentFetchKind::PerDocumentId { retrieve_vectors } => (1, 0, *retrieve_vectors), - DocumentFetchKind::Normal { limit, offset, retrieve_vectors, .. } => { - (*limit, *offset, *retrieve_vectors) - } - }; - Self { - timestamp: Some(OffsetDateTime::now_utc()), - user_agents: extract_user_agents(request).into_iter().collect(), - total_received: 1, - per_document_id: matches!(query, DocumentFetchKind::PerDocumentId { .. }), - per_filter: matches!(query, DocumentFetchKind::Normal { with_filter, .. } if *with_filter), - max_limit: limit, - max_offset: offset, - retrieve_vectors, - } - } - - /// Aggregate one [DocumentsFetchAggregator] into another. - pub fn aggregate(&mut self, other: Self) { - let Self { - timestamp, - user_agents, - total_received, - per_document_id, - per_filter, - max_limit, - max_offset, - retrieve_vectors, - } = other; - - if self.timestamp.is_none() { - self.timestamp = timestamp; - } - for user_agent in user_agents { - self.user_agents.insert(user_agent); - } - - self.total_received = self.total_received.saturating_add(total_received); - self.per_document_id |= per_document_id; - self.per_filter |= per_filter; - - self.max_limit = self.max_limit.max(max_limit); - self.max_offset = self.max_offset.max(max_offset); - - self.retrieve_vectors |= retrieve_vectors; - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - // if we had no timestamp it means we never encountered any events and - // thus we don't need to send this event. - let timestamp = self.timestamp?; - - Some(Track { - timestamp: Some(timestamp), - user: user.clone(), - event: event_name.to_string(), - properties: serde_json::to_value(self).ok()?, - ..Default::default() - }) - } -} - -#[derive(Default)] -pub struct SimilarAggregator { - timestamp: Option, - - // context - user_agents: HashSet, - - // requests - total_received: usize, - total_succeeded: usize, - time_spent: BinaryHeap, - - // filter - filter_with_geo_radius: bool, - filter_with_geo_bounding_box: bool, - // every time a request has a filter, this field must be incremented by the number of terms it contains - filter_sum_of_criteria_terms: usize, - // every time a request has a filter, this field must be incremented by one - filter_total_number_of_criteria: usize, - used_syntax: HashMap, - - // Whether a non-default embedder was specified - retrieve_vectors: bool, - - // pagination - max_limit: usize, - max_offset: usize, - - // formatting - max_attributes_to_retrieve: usize, - - // scoring - show_ranking_score: bool, - show_ranking_score_details: bool, - ranking_score_threshold: bool, -} - -impl SimilarAggregator { - #[allow(clippy::field_reassign_with_default)] - pub fn from_query(query: &SimilarQuery, request: &HttpRequest) -> Self { - let SimilarQuery { - id: _, - embedder: _, - offset, - limit, - attributes_to_retrieve: _, - retrieve_vectors, - show_ranking_score, - show_ranking_score_details, - filter, - ranking_score_threshold, - } = query; - - let mut ret = Self::default(); - ret.timestamp = Some(OffsetDateTime::now_utc()); - - ret.total_received = 1; - ret.user_agents = extract_user_agents(request).into_iter().collect(); - - if let Some(ref filter) = filter { - static RE: Lazy = Lazy::new(|| Regex::new("AND | OR").unwrap()); - ret.filter_total_number_of_criteria = 1; - - let syntax = match filter { - Value::String(_) => "string".to_string(), - Value::Array(values) => { - if values.iter().map(|v| v.to_string()).any(|s| RE.is_match(&s)) { - "mixed".to_string() - } else { - "array".to_string() - } - } - _ => "none".to_string(), - }; - // convert the string to a HashMap - ret.used_syntax.insert(syntax, 1); - - let stringified_filters = filter.to_string(); - ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius("); - ret.filter_with_geo_bounding_box = stringified_filters.contains("_geoBoundingBox("); - ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count(); - } - - ret.max_limit = *limit; - ret.max_offset = *offset; - - ret.show_ranking_score = *show_ranking_score; - ret.show_ranking_score_details = *show_ranking_score_details; - ret.ranking_score_threshold = ranking_score_threshold.is_some(); - - ret.retrieve_vectors = *retrieve_vectors; - - ret - } - - pub fn succeed(&mut self, result: &SimilarResult) { - let SimilarResult { id: _, hits: _, processing_time_ms, hits_info: _ } = result; - - self.total_succeeded = self.total_succeeded.saturating_add(1); - - self.time_spent.push(*processing_time_ms as usize); - } - - /// Aggregate one [SimilarAggregator] into another. - pub fn aggregate(&mut self, mut other: Self) { - let Self { - timestamp, - user_agents, - total_received, - total_succeeded, - ref mut time_spent, - filter_with_geo_radius, - filter_with_geo_bounding_box, - filter_sum_of_criteria_terms, - filter_total_number_of_criteria, - used_syntax, - max_limit, - max_offset, - max_attributes_to_retrieve, - show_ranking_score, - show_ranking_score_details, - ranking_score_threshold, - retrieve_vectors, - } = other; - - if self.timestamp.is_none() { - self.timestamp = timestamp; - } - - // context - for user_agent in user_agents.into_iter() { - self.user_agents.insert(user_agent); - } - - // request - self.total_received = self.total_received.saturating_add(total_received); - self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded); - self.time_spent.append(time_spent); - - // filter - self.filter_with_geo_radius |= filter_with_geo_radius; - self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box; - self.filter_sum_of_criteria_terms = - self.filter_sum_of_criteria_terms.saturating_add(filter_sum_of_criteria_terms); - self.filter_total_number_of_criteria = - self.filter_total_number_of_criteria.saturating_add(filter_total_number_of_criteria); - for (key, value) in used_syntax.into_iter() { - let used_syntax = self.used_syntax.entry(key).or_insert(0); - *used_syntax = used_syntax.saturating_add(value); - } - - self.retrieve_vectors |= retrieve_vectors; - - // pagination - self.max_limit = self.max_limit.max(max_limit); - self.max_offset = self.max_offset.max(max_offset); - - // formatting - self.max_attributes_to_retrieve = - self.max_attributes_to_retrieve.max(max_attributes_to_retrieve); - - // scoring - self.show_ranking_score |= show_ranking_score; - self.show_ranking_score_details |= show_ranking_score_details; - self.ranking_score_threshold |= ranking_score_threshold; - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - let Self { - timestamp, - user_agents, - total_received, - total_succeeded, - time_spent, - filter_with_geo_radius, - filter_with_geo_bounding_box, - filter_sum_of_criteria_terms, - filter_total_number_of_criteria, - used_syntax, - max_limit, - max_offset, - max_attributes_to_retrieve, - show_ranking_score, - show_ranking_score_details, - ranking_score_threshold, - retrieve_vectors, - } = self; - - if total_received == 0 { - None - } else { - // we get all the values in a sorted manner - let time_spent = time_spent.into_sorted_vec(); - // the index of the 99th percentage of value - let percentile_99th = time_spent.len() * 99 / 100; - // We are only interested by the slowest value of the 99th fastest results - let time_spent = time_spent.get(percentile_99th); - - let properties = json!({ - "user-agent": user_agents, - "requests": { - "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), - "total_succeeded": total_succeeded, - "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics - "total_received": total_received, - }, - "filter": { - "with_geoRadius": filter_with_geo_radius, - "with_geoBoundingBox": filter_with_geo_bounding_box, - "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64), - "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), - }, - "vector": { - "retrieve_vectors": retrieve_vectors, - }, - "pagination": { - "max_limit": max_limit, - "max_offset": max_offset, - }, - "formatting": { - "max_attributes_to_retrieve": max_attributes_to_retrieve, - }, - "scoring": { - "show_ranking_score": show_ranking_score, - "show_ranking_score_details": show_ranking_score_details, - "ranking_score_threshold": ranking_score_threshold, - }, - }); - - Some(Track { - timestamp, - user: user.clone(), - event: event_name.to_string(), - properties, - ..Default::default() - }) - } - } -} diff --git a/meilisearch/src/routes/indexes/settings.rs b/meilisearch/src/routes/indexes/settings.rs deleted file mode 100644 index aaf8673d0..000000000 --- a/meilisearch/src/routes/indexes/settings.rs +++ /dev/null @@ -1,901 +0,0 @@ -use actix_web::web::Data; -use actix_web::{web, HttpRequest, HttpResponse}; -use deserr::actix_web::AwebJson; -use index_scheduler::IndexScheduler; -use meilisearch_types::deserr::DeserrJsonError; -use meilisearch_types::error::ResponseError; -use meilisearch_types::facet_values_sort::FacetValuesSort; -use meilisearch_types::index_uid::IndexUid; -use meilisearch_types::milli::update::Setting; -use meilisearch_types::settings::{settings, RankingRuleView, SecretPolicy, Settings, Unchecked}; -use meilisearch_types::tasks::KindWithContent; -use serde_json::json; -use tracing::debug; - -use crate::analytics::Analytics; -use crate::extractors::authentication::policies::*; -use crate::extractors::authentication::GuardedData; -use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView}; -use crate::Opt; - -#[macro_export] -macro_rules! make_setting_route { - ($route:literal, $update_verb:ident, $type:ty, $err_ty:ty, $attr:ident, $camelcase_attr:literal, $analytics_var:ident, $analytics:expr) => { - pub mod $attr { - use actix_web::web::Data; - use actix_web::{web, HttpRequest, HttpResponse, Resource}; - use index_scheduler::IndexScheduler; - use meilisearch_types::error::ResponseError; - use meilisearch_types::index_uid::IndexUid; - use meilisearch_types::milli::update::Setting; - use meilisearch_types::settings::{settings, Settings}; - use meilisearch_types::tasks::KindWithContent; - use tracing::debug; - use $crate::analytics::Analytics; - use $crate::extractors::authentication::policies::*; - use $crate::extractors::authentication::GuardedData; - use $crate::extractors::sequential_extractor::SeqHandler; - use $crate::Opt; - use $crate::routes::{is_dry_run, get_task_id, SummarizedTaskView}; - - pub async fn delete( - index_scheduler: GuardedData< - ActionPolicy<{ actions::SETTINGS_UPDATE }>, - Data, - >, - index_uid: web::Path, - req: HttpRequest, - opt: web::Data, - ) -> Result { - let index_uid = IndexUid::try_from(index_uid.into_inner())?; - - let new_settings = Settings { $attr: Setting::Reset.into(), ..Default::default() }; - - let allow_index_creation = - index_scheduler.filters().allow_index_creation(&index_uid); - - let task = KindWithContent::SettingsUpdate { - index_uid: index_uid.to_string(), - new_settings: Box::new(new_settings), - is_deletion: true, - allow_index_creation, - }; - let uid = get_task_id(&req, &opt)?; - let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); - - debug!(returns = ?task, "Delete settings"); - Ok(HttpResponse::Accepted().json(task)) - } - - pub async fn update( - index_scheduler: GuardedData< - ActionPolicy<{ actions::SETTINGS_UPDATE }>, - Data, - >, - index_uid: actix_web::web::Path, - body: deserr::actix_web::AwebJson, $err_ty>, - req: HttpRequest, - opt: web::Data, - $analytics_var: web::Data, - ) -> std::result::Result { - let index_uid = IndexUid::try_from(index_uid.into_inner())?; - - let body = body.into_inner(); - debug!(parameters = ?body, "Update settings"); - - #[allow(clippy::redundant_closure_call)] - $analytics(&body, &req); - - let new_settings = Settings { - $attr: match body { - Some(inner_body) => Setting::Set(inner_body).into(), - None => Setting::Reset.into(), - }, - ..Default::default() - }; - - let new_settings = $crate::routes::indexes::settings::validate_settings( - new_settings, - &index_scheduler, - )?; - - let allow_index_creation = - index_scheduler.filters().allow_index_creation(&index_uid); - - let task = KindWithContent::SettingsUpdate { - index_uid: index_uid.to_string(), - new_settings: Box::new(new_settings), - is_deletion: false, - allow_index_creation, - }; - let uid = get_task_id(&req, &opt)?; - let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); - - debug!(returns = ?task, "Update settings"); - Ok(HttpResponse::Accepted().json(task)) - } - - pub async fn get( - index_scheduler: GuardedData< - ActionPolicy<{ actions::SETTINGS_GET }>, - Data, - >, - index_uid: actix_web::web::Path, - ) -> std::result::Result { - let index_uid = IndexUid::try_from(index_uid.into_inner())?; - - let index = index_scheduler.index(&index_uid)?; - let rtxn = index.read_txn()?; - let settings = settings(&index, &rtxn, meilisearch_types::settings::SecretPolicy::HideSecrets)?; - - debug!(returns = ?settings, "Update settings"); - - Ok(HttpResponse::Ok().json(settings.$attr)) - } - - pub fn resources() -> Resource { - Resource::new($route) - .route(web::get().to(SeqHandler(get))) - .route(web::$update_verb().to(SeqHandler(update))) - .route(web::delete().to(SeqHandler(delete))) - } - } - }; -} - -make_setting_route!( - "/filterable-attributes", - put, - std::collections::BTreeSet, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsFilterableAttributes, - >, - filterable_attributes, - "filterableAttributes", - analytics, - |setting: &Option>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "FilterableAttributes Updated".to_string(), - json!({ - "filterable_attributes": { - "total": setting.as_ref().map(|filter| filter.len()).unwrap_or(0), - "has_geo": setting.as_ref().map(|filter| filter.contains("_geo")).unwrap_or(false), - } - }), - Some(req), - ); - } -); - -make_setting_route!( - "/sortable-attributes", - put, - std::collections::BTreeSet, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsSortableAttributes, - >, - sortable_attributes, - "sortableAttributes", - analytics, - |setting: &Option>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "SortableAttributes Updated".to_string(), - json!({ - "sortable_attributes": { - "total": setting.as_ref().map(|sort| sort.len()), - "has_geo": setting.as_ref().map(|sort| sort.contains("_geo")), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/displayed-attributes", - put, - Vec, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsDisplayedAttributes, - >, - displayed_attributes, - "displayedAttributes", - analytics, - |displayed: &Option>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "DisplayedAttributes Updated".to_string(), - json!({ - "displayed_attributes": { - "total": displayed.as_ref().map(|displayed| displayed.len()), - "with_wildcard": displayed.as_ref().map(|displayed| displayed.iter().any(|displayed| displayed == "*")), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/typo-tolerance", - patch, - meilisearch_types::settings::TypoSettings, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsTypoTolerance, - >, - typo_tolerance, - "typoTolerance", - analytics, - |setting: &Option, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "TypoTolerance Updated".to_string(), - json!({ - "typo_tolerance": { - "enabled": setting.as_ref().map(|s| !matches!(s.enabled, Setting::Set(false))), - "disable_on_attributes": setting - .as_ref() - .and_then(|s| s.disable_on_attributes.as_ref().set().map(|m| !m.is_empty())), - "disable_on_words": setting - .as_ref() - .and_then(|s| s.disable_on_words.as_ref().set().map(|m| !m.is_empty())), - "min_word_size_for_one_typo": setting - .as_ref() - .and_then(|s| s.min_word_size_for_typos - .as_ref() - .set() - .map(|s| s.one_typo.set())) - .flatten(), - "min_word_size_for_two_typos": setting - .as_ref() - .and_then(|s| s.min_word_size_for_typos - .as_ref() - .set() - .map(|s| s.two_typos.set())) - .flatten(), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/searchable-attributes", - put, - Vec, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsSearchableAttributes, - >, - searchable_attributes, - "searchableAttributes", - analytics, - |setting: &Option>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "SearchableAttributes Updated".to_string(), - json!({ - "searchable_attributes": { - "total": setting.as_ref().map(|searchable| searchable.len()), - "with_wildcard": setting.as_ref().map(|searchable| searchable.iter().any(|searchable| searchable == "*")), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/stop-words", - put, - std::collections::BTreeSet, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsStopWords, - >, - stop_words, - "stopWords", - analytics, - |stop_words: &Option>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "StopWords Updated".to_string(), - json!({ - "stop_words": { - "total": stop_words.as_ref().map(|stop_words| stop_words.len()), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/non-separator-tokens", - put, - std::collections::BTreeSet, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsNonSeparatorTokens, - >, - non_separator_tokens, - "nonSeparatorTokens", - analytics, - |non_separator_tokens: &Option>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "nonSeparatorTokens Updated".to_string(), - json!({ - "non_separator_tokens": { - "total": non_separator_tokens.as_ref().map(|non_separator_tokens| non_separator_tokens.len()), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/separator-tokens", - put, - std::collections::BTreeSet, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsSeparatorTokens, - >, - separator_tokens, - "separatorTokens", - analytics, - |separator_tokens: &Option>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "separatorTokens Updated".to_string(), - json!({ - "separator_tokens": { - "total": separator_tokens.as_ref().map(|separator_tokens| separator_tokens.len()), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/dictionary", - put, - std::collections::BTreeSet, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsDictionary, - >, - dictionary, - "dictionary", - analytics, - |dictionary: &Option>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "dictionary Updated".to_string(), - json!({ - "dictionary": { - "total": dictionary.as_ref().map(|dictionary| dictionary.len()), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/synonyms", - put, - std::collections::BTreeMap>, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsSynonyms, - >, - synonyms, - "synonyms", - analytics, - |synonyms: &Option>>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "Synonyms Updated".to_string(), - json!({ - "synonyms": { - "total": synonyms.as_ref().map(|synonyms| synonyms.len()), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/distinct-attribute", - put, - String, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsDistinctAttribute, - >, - distinct_attribute, - "distinctAttribute", - analytics, - |distinct: &Option, req: &HttpRequest| { - use serde_json::json; - analytics.publish( - "DistinctAttribute Updated".to_string(), - json!({ - "distinct_attribute": { - "set": distinct.is_some(), - } - }), - Some(req), - ); - } -); - -make_setting_route!( - "/proximity-precision", - put, - meilisearch_types::settings::ProximityPrecisionView, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsProximityPrecision, - >, - proximity_precision, - "proximityPrecision", - analytics, - |precision: &Option, req: &HttpRequest| { - use serde_json::json; - analytics.publish( - "ProximityPrecision Updated".to_string(), - json!({ - "proximity_precision": { - "set": precision.is_some(), - "value": precision.unwrap_or_default(), - } - }), - Some(req), - ); - } -); - -make_setting_route!( - "/localized-attributes", - put, - Vec, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsLocalizedAttributes, - >, - localized_attributes, - "localizedAttributes", - analytics, - |rules: &Option>, req: &HttpRequest| { - use serde_json::json; - analytics.publish( - "LocalizedAttributesRules Updated".to_string(), - json!({ - "locales": rules.as_ref().map(|rules| rules.iter().flat_map(|rule| rule.locales.iter().cloned()).collect::>()) - }), - Some(req), - ); - } -); - -make_setting_route!( - "/ranking-rules", - put, - Vec, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsRankingRules, - >, - ranking_rules, - "rankingRules", - analytics, - |setting: &Option>, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "RankingRules Updated".to_string(), - json!({ - "ranking_rules": { - "words_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Words))), - "typo_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Typo))), - "proximity_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Proximity))), - "attribute_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Attribute))), - "sort_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Sort))), - "exactness_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Exactness))), - "values": setting.as_ref().map(|rr| rr.iter().filter(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Asc(_) | meilisearch_types::settings::RankingRuleView::Desc(_)) ).map(|x| x.to_string()).collect::>().join(", ")), - } - }), - Some(req), - ); - } -); - -make_setting_route!( - "/faceting", - patch, - meilisearch_types::settings::FacetingSettings, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsFaceting, - >, - faceting, - "faceting", - analytics, - |setting: &Option, req: &HttpRequest| { - use serde_json::json; - use meilisearch_types::facet_values_sort::FacetValuesSort; - - analytics.publish( - "Faceting Updated".to_string(), - json!({ - "faceting": { - "max_values_per_facet": setting.as_ref().and_then(|s| s.max_values_per_facet.set()), - "sort_facet_values_by_star_count": setting.as_ref().and_then(|s| { - s.sort_facet_values_by.as_ref().set().map(|s| s.iter().any(|(k, v)| k == "*" && v == &FacetValuesSort::Count)) - }), - "sort_facet_values_by_total": setting.as_ref().and_then(|s| s.sort_facet_values_by.as_ref().set().map(|s| s.len())), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/pagination", - patch, - meilisearch_types::settings::PaginationSettings, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsPagination, - >, - pagination, - "pagination", - analytics, - |setting: &Option, req: &HttpRequest| { - use serde_json::json; - - analytics.publish( - "Pagination Updated".to_string(), - json!({ - "pagination": { - "max_total_hits": setting.as_ref().and_then(|s| s.max_total_hits.set()), - }, - }), - Some(req), - ); - } -); - -make_setting_route!( - "/embedders", - patch, - std::collections::BTreeMap>, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsEmbedders, - >, - embedders, - "embedders", - analytics, - |setting: &Option>>, req: &HttpRequest| { - - - analytics.publish( - "Embedders Updated".to_string(), - serde_json::json!({"embedders": crate::routes::indexes::settings::embedder_analytics(setting.as_ref())}), - Some(req), - ); - } -); - -fn embedder_analytics( - setting: Option< - &std::collections::BTreeMap< - String, - Setting, - >, - >, -) -> serde_json::Value { - let mut sources = std::collections::HashSet::new(); - - if let Some(s) = &setting { - for source in s - .values() - .filter_map(|config| config.clone().set()) - .filter_map(|config| config.source.set()) - { - use meilisearch_types::milli::vector::settings::EmbedderSource; - match source { - EmbedderSource::OpenAi => sources.insert("openAi"), - EmbedderSource::HuggingFace => sources.insert("huggingFace"), - EmbedderSource::UserProvided => sources.insert("userProvided"), - EmbedderSource::Ollama => sources.insert("ollama"), - EmbedderSource::Rest => sources.insert("rest"), - }; - } - }; - - let document_template_used = setting.as_ref().map(|map| { - map.values() - .filter_map(|config| config.clone().set()) - .any(|config| config.document_template.set().is_some()) - }); - - let document_template_max_bytes = setting.as_ref().and_then(|map| { - map.values() - .filter_map(|config| config.clone().set()) - .filter_map(|config| config.document_template_max_bytes.set()) - .max() - }); - - let binary_quantization_used = setting.as_ref().map(|map| { - map.values() - .filter_map(|config| config.clone().set()) - .any(|config| config.binary_quantized.set().is_some()) - }); - - json!( - { - "total": setting.as_ref().map(|s| s.len()), - "sources": sources, - "document_template_used": document_template_used, - "document_template_max_bytes": document_template_max_bytes, - "binary_quantization_used": binary_quantization_used, - } - ) -} - -make_setting_route!( - "/search-cutoff-ms", - put, - u64, - meilisearch_types::deserr::DeserrJsonError< - meilisearch_types::error::deserr_codes::InvalidSettingsSearchCutoffMs, - >, - search_cutoff_ms, - "searchCutoffMs", - analytics, - |setting: &Option, req: &HttpRequest| { - analytics.publish( - "Search Cutoff Updated".to_string(), - serde_json::json!({"search_cutoff_ms": setting }), - Some(req), - ); - } -); - -macro_rules! generate_configure { - ($($mod:ident),*) => { - pub fn configure(cfg: &mut web::ServiceConfig) { - use crate::extractors::sequential_extractor::SeqHandler; - cfg.service( - web::resource("") - .route(web::patch().to(SeqHandler(update_all))) - .route(web::get().to(SeqHandler(get_all))) - .route(web::delete().to(SeqHandler(delete_all)))) - $(.service($mod::resources()))*; - } - }; -} - -generate_configure!( - filterable_attributes, - sortable_attributes, - displayed_attributes, - localized_attributes, - searchable_attributes, - distinct_attribute, - proximity_precision, - stop_words, - separator_tokens, - non_separator_tokens, - dictionary, - synonyms, - ranking_rules, - typo_tolerance, - pagination, - faceting, - embedders, - search_cutoff_ms -); - -pub async fn update_all( - index_scheduler: GuardedData, Data>, - index_uid: web::Path, - body: AwebJson, DeserrJsonError>, - req: HttpRequest, - opt: web::Data, - analytics: web::Data, -) -> Result { - let index_uid = IndexUid::try_from(index_uid.into_inner())?; - - let new_settings = body.into_inner(); - debug!(parameters = ?new_settings, "Update all settings"); - let new_settings = validate_settings(new_settings, &index_scheduler)?; - - analytics.publish( - "Settings Updated".to_string(), - json!({ - "ranking_rules": { - "words_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Words))), - "typo_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Typo))), - "proximity_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Proximity))), - "attribute_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Attribute))), - "sort_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Sort))), - "exactness_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Exactness))), - "values": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().filter(|s| !matches!(s, RankingRuleView::Asc(_) | RankingRuleView::Desc(_)) ).map(|x| x.to_string()).collect::>().join(", ")), - }, - "searchable_attributes": { - "total": new_settings.searchable_attributes.as_ref().set().map(|searchable| searchable.len()), - "with_wildcard": new_settings.searchable_attributes.as_ref().set().map(|searchable| searchable.iter().any(|searchable| searchable == "*")), - }, - "displayed_attributes": { - "total": new_settings.displayed_attributes.as_ref().set().map(|displayed| displayed.len()), - "with_wildcard": new_settings.displayed_attributes.as_ref().set().map(|displayed| displayed.iter().any(|displayed| displayed == "*")), - }, - "sortable_attributes": { - "total": new_settings.sortable_attributes.as_ref().set().map(|sort| sort.len()), - "has_geo": new_settings.sortable_attributes.as_ref().set().map(|sort| sort.iter().any(|s| s == "_geo")), - }, - "filterable_attributes": { - "total": new_settings.filterable_attributes.as_ref().set().map(|filter| filter.len()), - "has_geo": new_settings.filterable_attributes.as_ref().set().map(|filter| filter.iter().any(|s| s == "_geo")), - }, - "distinct_attribute": { - "set": new_settings.distinct_attribute.as_ref().set().is_some() - }, - "proximity_precision": { - "set": new_settings.proximity_precision.as_ref().set().is_some(), - "value": new_settings.proximity_precision.as_ref().set().copied().unwrap_or_default() - }, - "typo_tolerance": { - "enabled": new_settings.typo_tolerance - .as_ref() - .set() - .and_then(|s| s.enabled.as_ref().set()) - .copied(), - "disable_on_attributes": new_settings.typo_tolerance - .as_ref() - .set() - .and_then(|s| s.disable_on_attributes.as_ref().set().map(|m| !m.is_empty())), - "disable_on_words": new_settings.typo_tolerance - .as_ref() - .set() - .and_then(|s| s.disable_on_words.as_ref().set().map(|m| !m.is_empty())), - "min_word_size_for_one_typo": new_settings.typo_tolerance - .as_ref() - .set() - .and_then(|s| s.min_word_size_for_typos - .as_ref() - .set() - .map(|s| s.one_typo.set())) - .flatten(), - "min_word_size_for_two_typos": new_settings.typo_tolerance - .as_ref() - .set() - .and_then(|s| s.min_word_size_for_typos - .as_ref() - .set() - .map(|s| s.two_typos.set())) - .flatten(), - }, - "faceting": { - "max_values_per_facet": new_settings.faceting - .as_ref() - .set() - .and_then(|s| s.max_values_per_facet.as_ref().set()), - "sort_facet_values_by_star_count": new_settings.faceting - .as_ref() - .set() - .and_then(|s| { - s.sort_facet_values_by.as_ref().set().map(|s| s.iter().any(|(k, v)| k == "*" && v == &FacetValuesSort::Count)) - }), - "sort_facet_values_by_total": new_settings.faceting - .as_ref() - .set() - .and_then(|s| s.sort_facet_values_by.as_ref().set().map(|s| s.len())), - }, - "pagination": { - "max_total_hits": new_settings.pagination - .as_ref() - .set() - .and_then(|s| s.max_total_hits.as_ref().set()), - }, - "stop_words": { - "total": new_settings.stop_words.as_ref().set().map(|stop_words| stop_words.len()), - }, - "synonyms": { - "total": new_settings.synonyms.as_ref().set().map(|synonyms| synonyms.len()), - }, - "embedders": crate::routes::indexes::settings::embedder_analytics(new_settings.embedders.as_ref().set()), - "search_cutoff_ms": new_settings.search_cutoff_ms.as_ref().set(), - "locales": new_settings.localized_attributes.as_ref().set().map(|rules| rules.iter().flat_map(|rule| rule.locales.iter().cloned()).collect::>()), - }), - Some(&req), - ); - - let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid); - let index_uid = IndexUid::try_from(index_uid.into_inner())?.into_inner(); - let task = KindWithContent::SettingsUpdate { - index_uid, - new_settings: Box::new(new_settings), - is_deletion: false, - allow_index_creation, - }; - let uid = get_task_id(&req, &opt)?; - let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); - - debug!(returns = ?task, "Update all settings"); - Ok(HttpResponse::Accepted().json(task)) -} - -pub async fn get_all( - index_scheduler: GuardedData, Data>, - index_uid: web::Path, -) -> Result { - let index_uid = IndexUid::try_from(index_uid.into_inner())?; - - let index = index_scheduler.index(&index_uid)?; - let rtxn = index.read_txn()?; - let new_settings = settings(&index, &rtxn, SecretPolicy::HideSecrets)?; - debug!(returns = ?new_settings, "Get all settings"); - Ok(HttpResponse::Ok().json(new_settings)) -} - -pub async fn delete_all( - index_scheduler: GuardedData, Data>, - index_uid: web::Path, - req: HttpRequest, - opt: web::Data, -) -> Result { - let index_uid = IndexUid::try_from(index_uid.into_inner())?; - - let new_settings = Settings::cleared().into_unchecked(); - - let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid); - let index_uid = IndexUid::try_from(index_uid.into_inner())?.into_inner(); - let task = KindWithContent::SettingsUpdate { - index_uid, - new_settings: Box::new(new_settings), - is_deletion: true, - allow_index_creation, - }; - let uid = get_task_id(&req, &opt)?; - let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); - - debug!(returns = ?task, "Delete all settings"); - Ok(HttpResponse::Accepted().json(task)) -} - -fn validate_settings( - settings: Settings, - index_scheduler: &IndexScheduler, -) -> Result, ResponseError> { - if matches!(settings.embedders, Setting::Set(_)) { - index_scheduler.features().check_vector("Passing `embedders` in settings")? - } - Ok(settings.validate()?) -} diff --git a/meilitool/src/main.rs b/meilitool/src/main.rs deleted file mode 100644 index 0b82c394a..000000000 --- a/meilitool/src/main.rs +++ /dev/null @@ -1,752 +0,0 @@ -use std::fs::{read_dir, read_to_string, remove_file, File}; -use std::io::BufWriter; -use std::path::PathBuf; - -use anyhow::{bail, Context}; -use clap::{Parser, Subcommand}; -use dump::{DumpWriter, IndexMetadata}; -use file_store::FileStore; -use meilisearch_auth::AuthController; -use meilisearch_types::heed::types::{SerdeJson, Str}; -use meilisearch_types::heed::{Database, Env, EnvOpenOptions, RoTxn, RwTxn, Unspecified}; -use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader}; -use meilisearch_types::milli::index::{db_name, main_key}; -use meilisearch_types::milli::{obkv_to_json, BEU32}; -use meilisearch_types::tasks::{Status, Task}; -use meilisearch_types::versioning::{create_version_file, get_version, parse_version}; -use meilisearch_types::Index; -use time::macros::format_description; -use time::OffsetDateTime; -use uuid_codec::UuidCodec; - -mod uuid_codec; - -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct Cli { - /// The database path where the Meilisearch is running. - #[arg(long, default_value = "data.ms/")] - db_path: PathBuf, - - #[command(subcommand)] - command: Command, -} - -#[derive(Subcommand)] -enum Command { - /// Clears the task queue and make it empty. - /// - /// This command can be safely executed even if Meilisearch is running and processing tasks. - /// Once the task queue is empty you can restart Meilisearch and no more tasks must be visible, - /// even the ones that were processing. However, it's highly possible that you see the processing - /// tasks in the queue again with an associated internal error message. - ClearTaskQueue, - - /// Exports a dump from the Meilisearch database. - /// - /// Make sure to run this command when Meilisearch is not running or running but not processing tasks. - /// If tasks are being processed while a dump is being exported there are chances for the dump to be - /// malformed with missing tasks. - /// - /// TODO Verify this claim or make sure it cannot happen and we can export dumps - /// without caring about killing Meilisearch first! - ExportADump { - /// The directory in which the dump will be created. - #[arg(long, default_value = "dumps/")] - dump_dir: PathBuf, - - /// Skip dumping the enqueued or processing tasks. - /// - /// Can be useful when there are a lot of them and it is not particularly useful - /// to keep them. Note that only the enqueued tasks takes up space so skipping - /// the processed ones is not particularly interesting. - #[arg(long)] - skip_enqueued_tasks: bool, - }, - - /// Attempts to upgrade from one major version to the next without a dump. - /// - /// Make sure to run this commmand when Meilisearch is not running! - /// If Meilisearch is running while executing this command, the database could be corrupted - /// (contain data from both the old and the new versions) - /// - /// Supported upgrade paths: - /// - /// - v1.9.0 -> v1.10.0 - OfflineUpgrade { - #[arg(long)] - target_version: String, - }, -} - -fn main() -> anyhow::Result<()> { - let Cli { db_path, command } = Cli::parse(); - - let detected_version = get_version(&db_path).context("While checking the version file")?; - - match command { - Command::ClearTaskQueue => clear_task_queue(db_path), - Command::ExportADump { dump_dir, skip_enqueued_tasks } => { - export_a_dump(db_path, dump_dir, skip_enqueued_tasks) - } - Command::OfflineUpgrade { target_version } => { - let target_version = parse_version(&target_version).context("While parsing `--target-version`. Make sure `--target-version` is in the format MAJOR.MINOR.PATCH")?; - OfflineUpgrade { db_path, current_version: detected_version, target_version }.upgrade() - } - } -} - -struct OfflineUpgrade { - db_path: PathBuf, - current_version: (String, String, String), - target_version: (String, String, String), -} - -impl OfflineUpgrade { - fn upgrade(self) -> anyhow::Result<()> { - // TODO: if we make this process support more versions, introduce a more flexible way of checking for the version - // currently only supports v1.9 to v1.10 - let (current_major, current_minor, current_patch) = &self.current_version; - - match (current_major.as_str(), current_minor.as_str(), current_patch.as_str()) { - ("1", "9", _) => {} - _ => { - bail!("Unsupported current version {current_major}.{current_minor}.{current_patch}. Can only upgrade from v1.9") - } - } - - let (target_major, target_minor, target_patch) = &self.target_version; - - match (target_major.as_str(), target_minor.as_str(), target_patch.as_str()) { - ("1", "10", _) => {} - _ => { - bail!("Unsupported target version {target_major}.{target_minor}.{target_patch}. Can only upgrade to v1.10") - } - } - - println!("Upgrading from {current_major}.{current_minor}.{current_patch} to {target_major}.{target_minor}.{target_patch}"); - - self.v1_9_to_v1_10()?; - - println!("Writing VERSION file"); - - create_version_file(&self.db_path, target_major, target_minor, target_patch) - .context("while writing VERSION file after the upgrade")?; - - println!("Success"); - - Ok(()) - } - - fn v1_9_to_v1_10(&self) -> anyhow::Result<()> { - // 2 changes here - - // 1. date format. needs to be done before opening the Index - // 2. REST embedders. We don't support this case right now, so bail - - let index_scheduler_path = self.db_path.join("tasks"); - let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&index_scheduler_path) } - .with_context(|| { - format!("While trying to open {:?}", index_scheduler_path.display()) - })?; - - let mut sched_wtxn = env.write_txn()?; - - let index_mapping: Database = - try_opening_database(&env, &sched_wtxn, "index-mapping")?; - - let index_stats: Database = - try_opening_database(&env, &sched_wtxn, "index-stats").with_context(|| { - format!("While trying to open {:?}", index_scheduler_path.display()) - })?; - - let index_count = - index_mapping.len(&sched_wtxn).context("while reading the number of indexes")?; - - // FIXME: not ideal, we have to pre-populate all indexes to prevent double borrow of sched_wtxn - // 1. immutably for the iteration - // 2. mutably for updating index stats - let indexes: Vec<_> = index_mapping - .iter(&sched_wtxn)? - .map(|res| res.map(|(uid, uuid)| (uid.to_owned(), uuid))) - .collect(); - - let mut rest_embedders = Vec::new(); - - let mut unwrapped_indexes = Vec::new(); - - // check that update can take place - for (index_index, result) in indexes.into_iter().enumerate() { - let (uid, uuid) = result?; - let index_path = self.db_path.join("indexes").join(uuid.to_string()); - - println!( - "[{}/{index_count}]Checking that update can take place for `{uid}` at `{}`", - index_index + 1, - index_path.display() - ); - - let index_env = unsafe { - // FIXME: fetch the 25 magic number from the index file - EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| { - format!("while opening index {uid} at '{}'", index_path.display()) - })? - }; - - let index_txn = index_env.read_txn().with_context(|| { - format!( - "while obtaining a write transaction for index {uid} at {}", - index_path.display() - ) - })?; - - println!("\t- Checking for incompatible embedders (REST embedders)"); - let rest_embedders_for_index = find_rest_embedders(&uid, &index_env, &index_txn)?; - - if rest_embedders_for_index.is_empty() { - unwrapped_indexes.push((uid, uuid)); - } else { - // no need to add to unwrapped indexes because we'll exit early - rest_embedders.push((uid, rest_embedders_for_index)); - } - } - - if !rest_embedders.is_empty() { - let rest_embedders = rest_embedders - .into_iter() - .flat_map(|(index, embedders)| std::iter::repeat(index.clone()).zip(embedders)) - .map(|(index, embedder)| format!("\t- embedder `{embedder}` in index `{index}`")) - .collect::>() - .join("\n"); - bail!("The update cannot take place because there are REST embedder(s). Remove them before proceeding with the update:\n{rest_embedders}\n\n\ - The database has not been modified and is still a valid v1.9 database."); - } - - println!("Update can take place, updating"); - - for (index_index, (uid, uuid)) in unwrapped_indexes.into_iter().enumerate() { - let index_path = self.db_path.join("indexes").join(uuid.to_string()); - - println!( - "[{}/{index_count}]Updating index `{uid}` at `{}`", - index_index + 1, - index_path.display() - ); - - let index_env = unsafe { - // FIXME: fetch the 25 magic number from the index file - EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| { - format!("while opening index {uid} at '{}'", index_path.display()) - })? - }; - - let mut index_wtxn = index_env.write_txn().with_context(|| { - format!( - "while obtaining a write transaction for index `{uid}` at `{}`", - index_path.display() - ) - })?; - - println!("\t- Updating index stats"); - update_index_stats(index_stats, &uid, uuid, &mut sched_wtxn)?; - println!("\t- Updating date format"); - update_date_format(&uid, &index_env, &mut index_wtxn)?; - - index_wtxn.commit().with_context(|| { - format!( - "while committing the write txn for index `{uid}` at {}", - index_path.display() - ) - })?; - } - - sched_wtxn.commit().context("while committing the write txn for the index-scheduler")?; - - println!("Upgrading database succeeded"); - - Ok(()) - } -} - -pub mod v1_9 { - pub type FieldDistribution = std::collections::BTreeMap; - - /// The statistics that can be computed from an `Index` object. - #[derive(serde::Serialize, serde::Deserialize, Debug)] - pub struct IndexStats { - /// Number of documents in the index. - pub number_of_documents: u64, - /// Size taken up by the index' DB, in bytes. - /// - /// This includes the size taken by both the used and free pages of the DB, and as the free pages - /// are not returned to the disk after a deletion, this number is typically larger than - /// `used_database_size` that only includes the size of the used pages. - pub database_size: u64, - /// Size taken by the used pages of the index' DB, in bytes. - /// - /// As the DB backend does not return to the disk the pages that are not currently used by the DB, - /// this value is typically smaller than `database_size`. - pub used_database_size: u64, - /// Association of every field name with the number of times it occurs in the documents. - pub field_distribution: FieldDistribution, - /// Creation date of the index. - pub created_at: time::OffsetDateTime, - /// Date of the last update of the index. - pub updated_at: time::OffsetDateTime, - } - - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Deserialize, Serialize)] - pub struct IndexEmbeddingConfig { - pub name: String, - pub config: EmbeddingConfig, - } - - #[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)] - pub struct EmbeddingConfig { - /// Options of the embedder, specific to each kind of embedder - pub embedder_options: EmbedderOptions, - } - - /// Options of an embedder, specific to each kind of embedder. - #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] - pub enum EmbedderOptions { - HuggingFace(hf::EmbedderOptions), - OpenAi(openai::EmbedderOptions), - Ollama(ollama::EmbedderOptions), - UserProvided(manual::EmbedderOptions), - Rest(rest::EmbedderOptions), - } - - impl Default for EmbedderOptions { - fn default() -> Self { - Self::OpenAi(openai::EmbedderOptions { api_key: None, dimensions: None }) - } - } - - mod hf { - #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] - pub struct EmbedderOptions { - pub model: String, - pub revision: Option, - } - } - mod openai { - - #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] - pub struct EmbedderOptions { - pub api_key: Option, - pub dimensions: Option, - } - } - mod ollama { - #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] - pub struct EmbedderOptions { - pub embedding_model: String, - pub url: Option, - pub api_key: Option, - } - } - mod manual { - #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] - pub struct EmbedderOptions { - pub dimensions: usize, - } - } - mod rest { - #[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize, Hash)] - pub struct EmbedderOptions { - pub api_key: Option, - pub dimensions: Option, - pub url: String, - pub input_field: Vec, - // path to the array of embeddings - pub path_to_embeddings: Vec, - // shape of a single embedding - pub embedding_object: Vec, - } - } - - pub type OffsetDateTime = time::OffsetDateTime; -} - -pub mod v1_10 { - use crate::v1_9; - - pub type FieldDistribution = std::collections::BTreeMap; - - /// The statistics that can be computed from an `Index` object. - #[derive(serde::Serialize, serde::Deserialize, Debug)] - pub struct IndexStats { - /// Number of documents in the index. - pub number_of_documents: u64, - /// Size taken up by the index' DB, in bytes. - /// - /// This includes the size taken by both the used and free pages of the DB, and as the free pages - /// are not returned to the disk after a deletion, this number is typically larger than - /// `used_database_size` that only includes the size of the used pages. - pub database_size: u64, - /// Size taken by the used pages of the index' DB, in bytes. - /// - /// As the DB backend does not return to the disk the pages that are not currently used by the DB, - /// this value is typically smaller than `database_size`. - pub used_database_size: u64, - /// Association of every field name with the number of times it occurs in the documents. - pub field_distribution: FieldDistribution, - /// Creation date of the index. - #[serde(with = "time::serde::rfc3339")] - pub created_at: time::OffsetDateTime, - /// Date of the last update of the index. - #[serde(with = "time::serde::rfc3339")] - pub updated_at: time::OffsetDateTime, - } - - impl From for IndexStats { - fn from( - v1_9::IndexStats { - number_of_documents, - database_size, - used_database_size, - field_distribution, - created_at, - updated_at, - }: v1_9::IndexStats, - ) -> Self { - IndexStats { - number_of_documents, - database_size, - used_database_size, - field_distribution, - created_at, - updated_at, - } - } - } - - #[derive(serde::Serialize, serde::Deserialize)] - #[serde(transparent)] - pub struct OffsetDateTime(#[serde(with = "time::serde::rfc3339")] pub time::OffsetDateTime); -} - -fn update_index_stats( - index_stats: Database, - index_uid: &str, - index_uuid: uuid::Uuid, - sched_wtxn: &mut RwTxn, -) -> anyhow::Result<()> { - let ctx = || format!("while updating index stats for index `{index_uid}`"); - - let stats: Option = index_stats - .remap_data_type::>() - .get(sched_wtxn, &index_uuid) - .with_context(ctx)?; - - if let Some(stats) = stats { - let stats: v1_10::IndexStats = stats.into(); - - index_stats - .remap_data_type::>() - .put(sched_wtxn, &index_uuid, &stats) - .with_context(ctx)?; - } - - Ok(()) -} - -fn update_date_format( - index_uid: &str, - index_env: &Env, - index_wtxn: &mut RwTxn, -) -> anyhow::Result<()> { - let main = try_opening_poly_database(index_env, index_wtxn, db_name::MAIN) - .with_context(|| format!("while updating date format for index `{index_uid}`"))?; - - date_round_trip(index_wtxn, index_uid, main, main_key::CREATED_AT_KEY)?; - date_round_trip(index_wtxn, index_uid, main, main_key::UPDATED_AT_KEY)?; - - Ok(()) -} - -fn find_rest_embedders( - index_uid: &str, - index_env: &Env, - index_txn: &RoTxn, -) -> anyhow::Result> { - let main = try_opening_poly_database(index_env, index_txn, db_name::MAIN) - .with_context(|| format!("while checking REST embedders for index `{index_uid}`"))?; - - let mut rest_embedders = vec![]; - - for config in main - .remap_types::>>() - .get(index_txn, main_key::EMBEDDING_CONFIGS)? - .unwrap_or_default() - { - if let v1_9::EmbedderOptions::Rest(_) = config.config.embedder_options { - rest_embedders.push(config.name); - } - } - - Ok(rest_embedders) -} - -fn date_round_trip( - wtxn: &mut RwTxn, - index_uid: &str, - db: Database, - key: &str, -) -> anyhow::Result<()> { - let datetime = - db.remap_types::>().get(wtxn, key).with_context( - || format!("could not read `{key}` while updating date format for index `{index_uid}`"), - )?; - - if let Some(datetime) = datetime { - db.remap_types::>() - .put(wtxn, key, &v1_10::OffsetDateTime(datetime)) - .with_context(|| { - format!( - "could not write `{key}` while updating date format for index `{index_uid}`" - ) - })?; - } - - Ok(()) -} - -/// Clears the task queue located at `db_path`. -fn clear_task_queue(db_path: PathBuf) -> anyhow::Result<()> { - let path = db_path.join("tasks"); - let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&path) } - .with_context(|| format!("While trying to open {:?}", path.display()))?; - - eprintln!("Deleting tasks from the database..."); - - let mut wtxn = env.write_txn()?; - let all_tasks = try_opening_poly_database(&env, &wtxn, "all-tasks")?; - let total = all_tasks.len(&wtxn)?; - let status = try_opening_poly_database(&env, &wtxn, "status")?; - let kind = try_opening_poly_database(&env, &wtxn, "kind")?; - let index_tasks = try_opening_poly_database(&env, &wtxn, "index-tasks")?; - let canceled_by = try_opening_poly_database(&env, &wtxn, "canceled_by")?; - let enqueued_at = try_opening_poly_database(&env, &wtxn, "enqueued-at")?; - let started_at = try_opening_poly_database(&env, &wtxn, "started-at")?; - let finished_at = try_opening_poly_database(&env, &wtxn, "finished-at")?; - - try_clearing_poly_database(&mut wtxn, all_tasks, "all-tasks")?; - try_clearing_poly_database(&mut wtxn, status, "status")?; - try_clearing_poly_database(&mut wtxn, kind, "kind")?; - try_clearing_poly_database(&mut wtxn, index_tasks, "index-tasks")?; - try_clearing_poly_database(&mut wtxn, canceled_by, "canceled_by")?; - try_clearing_poly_database(&mut wtxn, enqueued_at, "enqueued-at")?; - try_clearing_poly_database(&mut wtxn, started_at, "started-at")?; - try_clearing_poly_database(&mut wtxn, finished_at, "finished-at")?; - - wtxn.commit().context("While committing the transaction")?; - - eprintln!("Successfully deleted {total} tasks from the tasks database!"); - eprintln!("Deleting the content files from disk..."); - - let mut count = 0usize; - let update_files = db_path.join("update_files"); - let entries = read_dir(&update_files).with_context(|| { - format!("While trying to read the content of {:?}", update_files.display()) - })?; - for result in entries { - match result { - Ok(ent) => match remove_file(ent.path()) { - Ok(_) => count += 1, - Err(e) => eprintln!("Error while deleting {:?}: {}", ent.path().display(), e), - }, - Err(e) => { - eprintln!("Error while reading a file in {:?}: {}", update_files.display(), e) - } - } - } - - eprintln!("Successfully deleted {count} content files from disk!"); - - Ok(()) -} - -fn try_opening_database( - env: &Env, - rtxn: &RoTxn, - db_name: &str, -) -> anyhow::Result> { - env.open_database(rtxn, Some(db_name)) - .with_context(|| format!("While opening the {db_name:?} database"))? - .with_context(|| format!("Missing the {db_name:?} database")) -} - -fn try_opening_poly_database( - env: &Env, - rtxn: &RoTxn, - db_name: &str, -) -> anyhow::Result> { - env.database_options() - .name(db_name) - .open(rtxn) - .with_context(|| format!("While opening the {db_name:?} poly database"))? - .with_context(|| format!("Missing the {db_name:?} poly database")) -} - -fn try_clearing_poly_database( - wtxn: &mut RwTxn, - database: Database, - db_name: &str, -) -> anyhow::Result<()> { - database.clear(wtxn).with_context(|| format!("While clearing the {db_name:?} database")) -} - -/// Exports a dump into the dump directory. -fn export_a_dump( - db_path: PathBuf, - dump_dir: PathBuf, - skip_enqueued_tasks: bool, -) -> Result<(), anyhow::Error> { - let started_at = OffsetDateTime::now_utc(); - - // 1. Extracts the instance UID from disk - let instance_uid_path = db_path.join("instance-uid"); - let instance_uid = match read_to_string(&instance_uid_path) { - Ok(content) => match content.trim().parse() { - Ok(uuid) => Some(uuid), - Err(e) => { - eprintln!("Impossible to parse instance-uid: {e}"); - None - } - }, - Err(e) => { - eprintln!("Impossible to read {}: {}", instance_uid_path.display(), e); - None - } - }; - - let dump = DumpWriter::new(instance_uid).context("While creating a new dump")?; - let file_store = - FileStore::new(db_path.join("update_files")).context("While opening the FileStore")?; - - let index_scheduler_path = db_path.join("tasks"); - let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&index_scheduler_path) } - .with_context(|| format!("While trying to open {:?}", index_scheduler_path.display()))?; - - eprintln!("Dumping the keys..."); - - // 2. dump the keys - let auth_store = AuthController::new(&db_path, &None) - .with_context(|| format!("While opening the auth store at {}", db_path.display()))?; - let mut dump_keys = dump.create_keys()?; - let mut count = 0; - for key in auth_store.list_keys()? { - dump_keys.push_key(&key)?; - count += 1; - } - dump_keys.flush()?; - - eprintln!("Successfully dumped {count} keys!"); - - let rtxn = env.read_txn()?; - let all_tasks: Database> = - try_opening_database(&env, &rtxn, "all-tasks")?; - let index_mapping: Database = - try_opening_database(&env, &rtxn, "index-mapping")?; - - if skip_enqueued_tasks { - eprintln!("Skip dumping the enqueued tasks..."); - } else { - eprintln!("Dumping the enqueued tasks..."); - - // 3. dump the tasks - let mut dump_tasks = dump.create_tasks_queue()?; - let mut count = 0; - for ret in all_tasks.iter(&rtxn)? { - let (_, t) = ret?; - let status = t.status; - let content_file = t.content_uuid(); - let mut dump_content_file = dump_tasks.push_task(&t.into())?; - - // 3.1. Dump the `content_file` associated with the task if there is one and the task is not finished yet. - if let Some(content_file_uuid) = content_file { - if status == Status::Enqueued { - let content_file = file_store.get_update(content_file_uuid)?; - - let reader = - DocumentsBatchReader::from_reader(content_file).with_context(|| { - format!("While reading content file {:?}", content_file_uuid) - })?; - - let (mut cursor, documents_batch_index) = reader.into_cursor_and_fields_index(); - while let Some(doc) = cursor.next_document().with_context(|| { - format!("While iterating on content file {:?}", content_file_uuid) - })? { - dump_content_file - .push_document(&obkv_to_object(doc, &documents_batch_index)?)?; - } - dump_content_file.flush()?; - count += 1; - } - } - } - dump_tasks.flush()?; - - eprintln!("Successfully dumped {count} enqueued tasks!"); - } - - eprintln!("Dumping the indexes..."); - - // 4. Dump the indexes - let mut count = 0; - for result in index_mapping.iter(&rtxn)? { - let (uid, uuid) = result?; - let index_path = db_path.join("indexes").join(uuid.to_string()); - let index = Index::new(EnvOpenOptions::new(), &index_path).with_context(|| { - format!("While trying to open the index at path {:?}", index_path.display()) - })?; - - let rtxn = index.read_txn()?; - let metadata = IndexMetadata { - uid: uid.to_owned(), - primary_key: index.primary_key(&rtxn)?.map(String::from), - created_at: index.created_at(&rtxn)?, - updated_at: index.updated_at(&rtxn)?, - }; - let mut index_dumper = dump.create_index(uid, &metadata)?; - - let fields_ids_map = index.fields_ids_map(&rtxn)?; - let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); - - // 4.1. Dump the documents - for ret in index.all_documents(&rtxn)? { - let (_id, doc) = ret?; - let document = obkv_to_json(&all_fields, &fields_ids_map, doc)?; - index_dumper.push_document(&document)?; - } - - // 4.2. Dump the settings - let settings = meilisearch_types::settings::settings( - &index, - &rtxn, - meilisearch_types::settings::SecretPolicy::RevealSecrets, - )?; - index_dumper.settings(&settings)?; - count += 1; - } - - eprintln!("Successfully dumped {count} indexes!"); - // We will not dump experimental feature settings - eprintln!("The tool is not dumping experimental features, please set them by hand afterward"); - - let dump_uid = started_at.format(format_description!( - "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" - )).unwrap(); - - let path = dump_dir.join(format!("{}.dump", dump_uid)); - let file = File::create(&path)?; - dump.persist_to(BufWriter::new(file))?; - - eprintln!("Dump exported at path {:?}", path.display()); - - Ok(()) -} diff --git a/workloads/search/embeddings-movies-subset-hf.json b/workloads/search/embeddings-movies-subset-hf.json index aeeecac59..36f45cfb9 100644 --- a/workloads/search/embeddings-movies-subset-hf.json +++ b/workloads/search/embeddings-movies-subset-hf.json @@ -77,7 +77,8 @@ "q": "puppy cute comforting movie", "limit": 100, "hybrid": { - "semanticRatio": 0.1 + "semanticRatio": 0.1, + "embedder": "default" } } }, @@ -91,7 +92,8 @@ "q": "puppy cute comforting movie", "limit": 100, "hybrid": { - "semanticRatio": 0.5 + "semanticRatio": 0.5, + "embedder": "default" } } }, @@ -105,7 +107,8 @@ "q": "puppy cute comforting movie", "limit": 100, "hybrid": { - "semanticRatio": 0.9 + "semanticRatio": 0.9, + "embedder": "default" } } }, @@ -119,7 +122,8 @@ "q": "puppy cute comforting movie", "limit": 100, "hybrid": { - "semanticRatio": 1.0 + "semanticRatio": 1.0, + "embedder": "default" } } }, @@ -133,7 +137,8 @@ "q": "shrek", "limit": 100, "hybrid": { - "semanticRatio": 1.0 + "semanticRatio": 1.0, + "embedder": "default" } } }, @@ -147,7 +152,8 @@ "q": "shrek", "limit": 100, "hybrid": { - "semanticRatio": 0.5 + "semanticRatio": 0.5, + "embedder": "default" } } }, @@ -161,7 +167,8 @@ "q": "shrek", "limit": 100, "hybrid": { - "semanticRatio": 0.1 + "semanticRatio": 0.1, + "embedder": "default" } } },