From 6d4981ec250fd9e09389681022a4c96ee5497215 Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Fri, 23 Jun 2023 07:24:25 +0800 Subject: [PATCH 01/54] Expose lastUpdate and isIndexing in /stats endpoint --- index-scheduler/src/lib.rs | 24 ++++++++++++++++++++++++ meilisearch/src/metrics.rs | 10 ++++++++++ meilisearch/src/routes/metrics.rs | 3 +++ 3 files changed, 37 insertions(+) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 0f0ecf52a..c1b2deb27 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -794,6 +794,12 @@ impl IndexScheduler { Ok(res) } + // Return true if there is at least one task that is processing. + pub fn is_task_processing(&self) -> Result { + let processing_tasks = self.processing_tasks.read().unwrap().processing.len(); + Ok(processing_tasks > 0) + } + /// Return true iff there is at least one task associated with this index /// that is processing. pub fn is_index_processing(&self, index: &str) -> Result { @@ -1808,6 +1814,24 @@ mod tests { snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_third_task"); } + #[test] + fn test_task_is_processing() { + let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); + + let (file0, documents_count0) = sample_documents(&index_scheduler, 0, 0); + file0.persist().unwrap(); + + let _ = index_scheduler + .register(replace_document_import_task("catto", None, 0, documents_count0)) + .unwrap(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task"); + + handle.advance_till([Start, BatchCreated, InsideProcessBatch]); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_task_processing"); + + assert_eq!(index_scheduler.is_task_processing().unwrap(), true); + } + /// We send a lot of tasks but notify the tasks scheduler only once as /// we send them very fast, we must make sure that they are all processed. #[test] diff --git a/meilisearch/src/metrics.rs b/meilisearch/src/metrics.rs index 79332f360..009c3e472 100644 --- a/meilisearch/src/metrics.rs +++ b/meilisearch/src/metrics.rs @@ -50,4 +50,14 @@ lazy_static! { &["kind", "value"] ) .expect("Can't create a metric"); + pub static ref MEILISEARCH_LAST_UPDATE: IntGauge = register_int_gauge!(opts!( + "meilisearch_last_update", + "Meilisearch Last Update" + )) + .expect("Can't create a metric"); + pub static ref MEILISEARCH_IS_INDEXING: IntGauge = register_int_gauge!(opts!( + "meilisearch_is_indexing", + "Meilisearch Is Indexing" + )) + .expect("Can't create a metric"); } diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index a7d41e33e..f6ee3291b 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -48,6 +48,9 @@ pub async fn get_metrics( } } + crate::metrics::MEILISEARCH_LAST_UPDATE.set(response.last_update.unwrap().unix_timestamp() as i64); + crate::metrics::MEILISEARCH_IS_INDEXING.set(index_scheduler.is_task_processing().unwrap() as i64); + let encoder = TextEncoder::new(); let mut buffer = vec![]; encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics"); From e3fc7112bc3b449ffdcc5d3d62a4ef2ea07fc43e Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Thu, 29 Jun 2023 11:46:47 +0800 Subject: [PATCH 02/54] use `RoaringBitmap::is_empty` instead --- index-scheduler/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index c1b2deb27..fc017395e 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -796,8 +796,7 @@ impl IndexScheduler { // Return true if there is at least one task that is processing. pub fn is_task_processing(&self) -> Result { - let processing_tasks = self.processing_tasks.read().unwrap().processing.len(); - Ok(processing_tasks > 0) + Ok(self.processing_tasks.read().unwrap().processing.is_empty()) } /// Return true iff there is at least one task associated with this index From a5a31667b01f532473d8bd0218c5c805d5266d96 Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Fri, 30 Jun 2023 11:28:18 +0800 Subject: [PATCH 03/54] fix converse result of is_task_processing() --- index-scheduler/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index fc017395e..1f984e733 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -796,7 +796,7 @@ impl IndexScheduler { // Return true if there is at least one task that is processing. pub fn is_task_processing(&self) -> Result { - Ok(self.processing_tasks.read().unwrap().processing.is_empty()) + Ok(!self.processing_tasks.read().unwrap().processing.is_empty()) } /// Return true iff there is at least one task associated with this index From 3bdf01bc1c11dea3b38f2ed3cac78aeb89402272 Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Fri, 30 Jun 2023 17:39:23 +0800 Subject: [PATCH 04/54] Fix failed test --- index-scheduler/src/lib.rs | 10 +--------- meilisearch/src/routes/metrics.rs | 6 ++++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 1f984e733..0594e15e0 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -1817,17 +1817,9 @@ mod tests { fn test_task_is_processing() { let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); - let (file0, documents_count0) = sample_documents(&index_scheduler, 0, 0); - file0.persist().unwrap(); - - let _ = index_scheduler - .register(replace_document_import_task("catto", None, 0, documents_count0)) - .unwrap(); + index_scheduler.register(index_creation_task("index_a", "id")).unwrap(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task"); - handle.advance_till([Start, BatchCreated, InsideProcessBatch]); - snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_task_processing"); - assert_eq!(index_scheduler.is_task_processing().unwrap(), true); } diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index f6ee3291b..d35778881 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -48,8 +48,10 @@ pub async fn get_metrics( } } - crate::metrics::MEILISEARCH_LAST_UPDATE.set(response.last_update.unwrap().unix_timestamp() as i64); - crate::metrics::MEILISEARCH_IS_INDEXING.set(index_scheduler.is_task_processing().unwrap() as i64); + crate::metrics::MEILISEARCH_LAST_UPDATE + .set(response.last_update.unwrap().unix_timestamp()); + crate::metrics::MEILISEARCH_IS_INDEXING + .set(index_scheduler.is_task_processing().unwrap() as i64); let encoder = TextEncoder::new(); let mut buffer = vec![]; From 9859e65d2fa229a5a95862630cfcbf3610b5ac39 Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Sat, 1 Jul 2023 09:32:50 +0800 Subject: [PATCH 05/54] fix tests --- index-scheduler/src/lib.rs | 4 ++-- meilisearch/src/routes/metrics.rs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 0594e15e0..a9affed0b 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -1815,12 +1815,12 @@ mod tests { #[test] fn test_task_is_processing() { - let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); + let (index_scheduler, _handle) = IndexScheduler::test(true, vec![]); index_scheduler.register(index_creation_task("index_a", "id")).unwrap(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task"); - assert_eq!(index_scheduler.is_task_processing().unwrap(), true); + assert!(index_scheduler.is_task_processing().unwrap()); } /// We send a lot of tasks but notify the tasks scheduler only once as diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index d35778881..534c0c500 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -48,8 +48,7 @@ pub async fn get_metrics( } } - crate::metrics::MEILISEARCH_LAST_UPDATE - .set(response.last_update.unwrap().unix_timestamp()); + crate::metrics::MEILISEARCH_LAST_UPDATE.set(response.last_update.unwrap().unix_timestamp()); crate::metrics::MEILISEARCH_IS_INDEXING .set(index_scheduler.is_task_processing().unwrap() as i64); From 71500a4e150f80be90ada52e1e7aa316c626cc93 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Mon, 3 Jul 2023 11:20:43 +0200 Subject: [PATCH 06/54] Update tests --- index-scheduler/src/lib.rs | 3 +- .../registered_a_task.snap | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index a9affed0b..5bab13a2d 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -1815,11 +1815,12 @@ mod tests { #[test] fn test_task_is_processing() { - let (index_scheduler, _handle) = IndexScheduler::test(true, vec![]); + let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); index_scheduler.register(index_creation_task("index_a", "id")).unwrap(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task"); + handle.advance_till([Start, BatchCreated]); assert!(index_scheduler.is_task_processing().unwrap()); } diff --git a/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap b/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap new file mode 100644 index 000000000..f17bfe38f --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap @@ -0,0 +1,36 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +---------------------------------------------------------------------- +### Status: +enqueued [0,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [0,] +---------------------------------------------------------------------- +### Index Tasks: +index_a [0,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + From 5387cf17186dfd977275a5e30fc406d96f1e698b Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 3 Jul 2023 15:21:58 +0200 Subject: [PATCH 07/54] Don't unwrap in case of error/missing last_update field --- meilisearch/src/metrics.rs | 16 ++++++---------- meilisearch/src/routes/metrics.rs | 7 ++++--- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/meilisearch/src/metrics.rs b/meilisearch/src/metrics.rs index 009c3e472..2becdae76 100644 --- a/meilisearch/src/metrics.rs +++ b/meilisearch/src/metrics.rs @@ -50,14 +50,10 @@ lazy_static! { &["kind", "value"] ) .expect("Can't create a metric"); - pub static ref MEILISEARCH_LAST_UPDATE: IntGauge = register_int_gauge!(opts!( - "meilisearch_last_update", - "Meilisearch Last Update" - )) - .expect("Can't create a metric"); - pub static ref MEILISEARCH_IS_INDEXING: IntGauge = register_int_gauge!(opts!( - "meilisearch_is_indexing", - "Meilisearch Is Indexing" - )) - .expect("Can't create a metric"); + pub static ref MEILISEARCH_LAST_UPDATE: IntGauge = + register_int_gauge!(opts!("meilisearch_last_update", "Meilisearch Last Update")) + .expect("Can't create a metric"); + pub static ref MEILISEARCH_IS_INDEXING: IntGauge = + register_int_gauge!(opts!("meilisearch_is_indexing", "Meilisearch Is Indexing")) + .expect("Can't create a metric"); } diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index 534c0c500..93d07e67c 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -48,9 +48,10 @@ pub async fn get_metrics( } } - crate::metrics::MEILISEARCH_LAST_UPDATE.set(response.last_update.unwrap().unix_timestamp()); - crate::metrics::MEILISEARCH_IS_INDEXING - .set(index_scheduler.is_task_processing().unwrap() as i64); + if let Some(last_update) = response.last_update { + crate::metrics::MEILISEARCH_LAST_UPDATE.set(last_update.unix_timestamp()); + } + crate::metrics::MEILISEARCH_IS_INDEXING.set(index_scheduler.is_task_processing()? as i64); let encoder = TextEncoder::new(); let mut buffer = vec![]; From 202183adf806a15e1406227e32b2beed3d01f22c Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 6 Jun 2023 12:58:50 +0200 Subject: [PATCH 08/54] update the total_received properties of multiple events --- meilisearch/src/analytics/segment_analytics.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 25aa20a9a..17170b582 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -1145,6 +1145,7 @@ pub struct DocumentsDeletionAggregator { #[serde(rename = "user-agent")] user_agents: HashSet, + #[serde(rename = "requests.total_received")] total_received: usize, per_document_id: bool, clear_all: bool, @@ -1295,6 +1296,7 @@ pub struct HealthAggregator { #[serde(rename = "user-agent")] user_agents: HashSet, + #[serde(rename = "requests.total_received")] total_received: usize, } @@ -1345,7 +1347,7 @@ pub struct DocumentsFetchAggregator { #[serde(rename = "user-agent")] user_agents: HashSet, - #[serde(rename = "requests.max_limit")] + #[serde(rename = "requests.total_received")] total_received: usize, // a call on ../documents/:doc_id From d1ff631df841cae2aac44120fb64be08471195d3 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 3 Jul 2023 18:51:42 +0200 Subject: [PATCH 09/54] Replace the atty dependency with the is-terminal one --- Cargo.lock | 58 +++++++++++++++++++++++++++++------------ meilisearch/Cargo.toml | 16 +++--------- meilisearch/src/main.rs | 9 +++---- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a59e2ea8..bcfe40cd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bytes", "futures-core", "futures-sink", @@ -47,7 +47,7 @@ dependencies = [ "actix-utils", "ahash 0.8.3", "base64 0.21.2", - "bitflags", + "bitflags 1.3.2", "brotli", "bytes", "bytestring", @@ -527,6 +527,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "block-buffer" version = "0.10.4" @@ -773,7 +779,7 @@ version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ - "bitflags", + "bitflags 1.3.2", "clap_lex 0.2.4", "indexmap", "textwrap", @@ -798,7 +804,7 @@ checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" dependencies = [ "anstream", "anstyle", - "bitflags", + "bitflags 1.3.2", "clap_lex 0.5.0", "strsim", ] @@ -1676,7 +1682,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "libgit2-sys", "log", @@ -2058,13 +2064,12 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" dependencies = [ "hermit-abi 0.3.1", - "io-lifetimes", - "rustix 0.37.19", + "rustix 0.38.2", "windows-sys 0.48.0", ] @@ -2161,9 +2166,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.144" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libgit2-sys" @@ -2432,6 +2437,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + [[package]] name = "lmdb-rkv-sys" version = "0.15.1" @@ -2545,7 +2556,6 @@ dependencies = [ "assert-json-diff", "async-stream", "async-trait", - "atty", "brotli", "bstr", "byte-unit", @@ -2567,6 +2577,7 @@ dependencies = [ "index-scheduler", "indexmap", "insta", + "is-terminal", "itertools", "jsonwebtoken", "lazy_static", @@ -3226,7 +3237,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69" dependencies = [ - "bitflags", + "bitflags 1.3.2", "byteorder", "hex", "lazy_static", @@ -3333,7 +3344,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -3342,7 +3353,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -3484,7 +3495,7 @@ version = "0.36.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -3498,7 +3509,7 @@ version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -3506,6 +3517,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustix" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4" +dependencies = [ + "bitflags 2.3.3", + "errno", + "libc", + "linux-raw-sys 0.4.3", + "windows-sys 0.48.0", +] + [[package]] name = "rustls" version = "0.20.8" diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index 7d6601ac5..3fbc21d82 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -19,6 +19,7 @@ actix-http = { version = "3.3.1", default-features = false, features = [ "compress-gzip", "rustls", ] } +actix-utils = "3.0.1" actix-web = { version = "4.3.1", default-features = false, features = [ "macros", "compress-brotli", @@ -50,6 +51,7 @@ futures-util = "0.3.28" http = "0.2.9" index-scheduler = { path = "../index-scheduler" } indexmap = { version = "1.9.3", features = ["serde-1"] } +is-terminal = "0.4.8" itertools = "0.10.5" jsonwebtoken = "8.3.0" lazy_static = "1.4.0" @@ -100,8 +102,6 @@ uuid = { version = "1.3.1", features = ["serde", "v4"] } walkdir = "2.3.3" yaup = "0.2.1" serde_urlencoded = "0.7.1" -actix-utils = "3.0.1" -atty = "0.2.14" termcolor = "1.2.0" [dev-dependencies] @@ -133,17 +133,7 @@ zip = { version = "0.6.4", optional = true } [features] default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"] analytics = ["segment"] -mini-dashboard = [ - "actix-web-static-files", - "static-files", - "anyhow", - "cargo_toml", - "hex", - "reqwest", - "sha-1", - "tempfile", - "zip", -] +mini-dashboard = ["actix-web-static-files", "static-files", "anyhow", "cargo_toml", "hex", "reqwest", "sha-1", "tempfile", "zip"] chinese = ["meilisearch-types/chinese"] hebrew = ["meilisearch-types/hebrew"] japanese = ["meilisearch-types/japanese"] diff --git a/meilisearch/src/main.rs b/meilisearch/src/main.rs index d6b9f027d..b0ca9dd7b 100644 --- a/meilisearch/src/main.rs +++ b/meilisearch/src/main.rs @@ -1,5 +1,5 @@ use std::env; -use std::io::Write; +use std::io::{stderr, Write}; use std::path::PathBuf; use std::sync::Arc; @@ -7,6 +7,7 @@ use actix_web::http::KeepAlive; use actix_web::web::Data; use actix_web::HttpServer; use index_scheduler::IndexScheduler; +use is_terminal::IsTerminal; use meilisearch::analytics::Analytics; use meilisearch::{analytics, create_app, prototype_name, setup_meilisearch, Opt}; use meilisearch_auth::{generate_master_key, AuthController, MASTER_KEY_MIN_SIZE}; @@ -197,8 +198,7 @@ const WARNING_BG_COLOR: Option = Some(Color::Ansi256(178)); const WARNING_FG_COLOR: Option = Some(Color::Ansi256(0)); fn print_master_key_too_short_warning() { - let choice = - if atty::is(atty::Stream::Stderr) { ColorChoice::Auto } else { ColorChoice::Never }; + let choice = if stderr().is_terminal() { ColorChoice::Auto } else { ColorChoice::Never }; let mut stderr = StandardStream::stderr(choice); stderr .set_color( @@ -223,8 +223,7 @@ fn print_master_key_too_short_warning() { } fn print_missing_master_key_warning() { - let choice = - if atty::is(atty::Stream::Stderr) { ColorChoice::Auto } else { ColorChoice::Never }; + let choice = if stderr().is_terminal() { ColorChoice::Auto } else { ColorChoice::Never }; let mut stderr = StandardStream::stderr(choice); stderr .set_color( From e7f8daaf86601d557947e0f27c2a56cc81da96b8 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 3 Jul 2023 17:09:29 +0200 Subject: [PATCH 10/54] Update criterion to 0.5.1 to remove the atty dependency --- Cargo.lock | 69 ++++------------------------------- benchmarks/Cargo.toml | 2 +- flatten-serde-json/Cargo.toml | 2 +- json-depth-checker/Cargo.toml | 2 +- 4 files changed, 11 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bcfe40cd3..bb6105741 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -428,17 +428,6 @@ dependencies = [ "critical-section", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -773,18 +762,6 @@ dependencies = [ "inout", ] -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "bitflags 1.3.2", - "clap_lex 0.2.4", - "indexmap", - "textwrap", -] - [[package]] name = "clap" version = "4.3.0" @@ -805,7 +782,7 @@ dependencies = [ "anstream", "anstyle", "bitflags 1.3.2", - "clap_lex 0.5.0", + "clap_lex", "strsim", ] @@ -821,15 +798,6 @@ dependencies = [ "syn 2.0.18", ] -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - [[package]] name = "clap_lex" version = "0.5.0" @@ -935,19 +903,19 @@ dependencies = [ [[package]] name = "criterion" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", - "atty", "cast", "ciborium", - "clap 3.2.25", + "clap", "criterion-plot", + "is-terminal", "itertools", - "lazy_static", "num-traits", + "once_cell", "oorandom", "plotters", "rayon", @@ -1614,7 +1582,7 @@ name = "fuzzers" version = "1.3.0" dependencies = [ "arbitrary", - "clap 4.3.0", + "clap", "fastrand", "milli", "serde", @@ -1811,15 +1779,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -2561,7 +2520,7 @@ dependencies = [ "byte-unit", "bytes", "cargo_toml", - "clap 4.3.0", + "clap", "crossbeam-channel", "deserr", "dump", @@ -2933,12 +2892,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "os_str_bytes" -version = "6.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" - [[package]] name = "page_size" version = "0.4.2" @@ -3973,12 +3926,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" - [[package]] name = "thiserror" version = "1.0.40" diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index aa4229e05..330947b34 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -18,7 +18,7 @@ mimalloc = { version = "0.1.36", default-features = false } serde_json = { version = "1.0.95", features = ["preserve_order"] } [dev-dependencies] -criterion = { version = "0.4.0", features = ["html_reports"] } +criterion = { version = "0.5.1", features = ["html_reports"] } rand = "0.8.5" rand_chacha = "0.3.1" roaring = "0.10.1" diff --git a/flatten-serde-json/Cargo.toml b/flatten-serde-json/Cargo.toml index 1ad00b0b3..7b498ec4f 100644 --- a/flatten-serde-json/Cargo.toml +++ b/flatten-serde-json/Cargo.toml @@ -16,7 +16,7 @@ license.workspace = true serde_json = "1.0" [dev-dependencies] -criterion = { version = "0.4.0", features = ["html_reports"] } +criterion = { version = "0.5.1", features = ["html_reports"] } [[bench]] name = "benchmarks" diff --git a/json-depth-checker/Cargo.toml b/json-depth-checker/Cargo.toml index 01b2d03a7..b8162357b 100644 --- a/json-depth-checker/Cargo.toml +++ b/json-depth-checker/Cargo.toml @@ -15,7 +15,7 @@ license.workspace = true serde_json = "1.0" [dev-dependencies] -criterion = "0.4.0" +criterion = "0.5.1" [[bench]] name = "depth" From a442af6a7c8d225790c860ca55c35b368b03a2fa Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 3 Jul 2023 17:10:05 +0200 Subject: [PATCH 11/54] Update the features of the either dependency to compile milli successfully --- milli/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/milli/Cargo.toml b/milli/Cargo.toml index ce3070e5d..bc1d9b7ee 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -21,7 +21,7 @@ charabia = { version = "0.8.1", default-features = false } concat-arrays = "0.1.2" crossbeam-channel = "0.5.8" deserr = "0.5.0" -either = "1.8.1" +either = { version = "1.8.1", features = ["serde"] } flatten-serde-json = { path = "../flatten-serde-json" } fst = "0.4.7" fxhash = "0.2.1" From 48409c91839c2a02c413b2774652cbde6553f531 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 4 Jul 2023 15:53:38 +0200 Subject: [PATCH 12/54] Add missing exactness.matchingWords, exactness.maxMatchingWords --- milli/src/score_details.rs | 54 +++++++++++++++---- .../new/ranking_rule_graph/exactness/mod.rs | 4 +- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/milli/src/score_details.rs b/milli/src/score_details.rs index 37b486047..0235436d8 100644 --- a/milli/src/score_details.rs +++ b/milli/src/score_details.rs @@ -10,7 +10,7 @@ pub enum ScoreDetails { Fid(Rank), Position(Rank), ExactAttribute(ExactAttribute), - Exactness(Rank), + ExactWords(ExactWords), Sort(Sort), GeoSort(GeoSort), } @@ -28,7 +28,7 @@ impl ScoreDetails { ScoreDetails::Fid(details) => Some(*details), ScoreDetails::Position(details) => Some(*details), ScoreDetails::ExactAttribute(details) => Some(details.rank()), - ScoreDetails::Exactness(details) => Some(*details), + ScoreDetails::ExactWords(details) => Some(details.rank()), ScoreDetails::Sort(_) => None, ScoreDetails::GeoSort(_) => None, } @@ -117,7 +117,7 @@ impl ScoreDetails { details_map.insert("exactness".into(), exactness_details); order += 1; } - ScoreDetails::Exactness(details) => { + ScoreDetails::ExactWords(details) => { // For now, exactness is a virtual rule always preceded by the "ExactAttribute" rule let exactness_details = details_map .get_mut("exactness") @@ -129,9 +129,16 @@ impl ScoreDetails { == &serde_json::json!(ExactAttribute::NoExactMatch) { let score = Rank::global_score( - [ExactAttribute::NoExactMatch.rank(), *details].iter().copied(), + [ExactAttribute::NoExactMatch.rank(), details.rank()].iter().copied(), ); - *exactness_details.get_mut("score").expect("missing score") = score.into(); + // tiny detail, but we want the score to be the last displayed field, + // so we're removing it here, adding the other fields, then adding the new score + exactness_details.remove("score"); + exactness_details + .insert("matchingWords".into(), details.matching_words.into()); + exactness_details + .insert("maxMatchingWords".into(), details.max_matching_words.into()); + exactness_details.insert("score".into(), score.into()); } // do not update the order since this was already done by exactAttribute } @@ -209,8 +216,34 @@ impl Words { Rank { rank: self.matching_words, max_rank: self.max_matching_words } } - pub(crate) fn from_rank(rank: Rank) -> Words { - Words { matching_words: rank.rank, max_matching_words: rank.max_rank } + pub(crate) fn from_rank(rank: Rank) -> Self { + Self { matching_words: rank.rank, max_matching_words: rank.max_rank } + } +} + +/// Structure that is super similar to [`Words`], but whose semantics is a bit distinct. +/// +/// In exactness, the number of matching words can actually be 0 with a non-zero score, +/// if no words from the query appear exactly in the document. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ExactWords { + pub matching_words: u32, + pub max_matching_words: u32, +} + +impl ExactWords { + pub fn rank(&self) -> Rank { + // 0 matching words means last rank (1) + Rank { rank: self.matching_words + 1, max_rank: self.max_matching_words + 1 } + } + + pub(crate) fn from_rank(rank: Rank) -> Self { + // last rank (1) means that 0 words from the query appear exactly in the document. + // first rank (max_rank) means that (max_rank - 1) words from the query appear exactly in the document. + Self { + matching_words: rank.rank.saturating_sub(1), + max_matching_words: rank.max_rank.saturating_sub(1), + } } } @@ -223,7 +256,7 @@ pub struct Typo { impl Typo { pub fn rank(&self) -> Rank { Rank { - rank: self.max_typo_count - self.typo_count + 1, + rank: (self.max_typo_count + 1).saturating_sub(self.typo_count), max_rank: (self.max_typo_count + 1), } } @@ -236,7 +269,10 @@ impl Typo { // rank + typo = max_rank // typo = max_rank - rank pub fn from_rank(rank: Rank) -> Typo { - Typo { typo_count: rank.max_rank - rank.rank, max_typo_count: rank.max_rank - 1 } + Typo { + typo_count: rank.max_rank.saturating_sub(rank.rank), + max_typo_count: rank.max_rank.saturating_sub(1), + } } } diff --git a/milli/src/search/new/ranking_rule_graph/exactness/mod.rs b/milli/src/search/new/ranking_rule_graph/exactness/mod.rs index 0a84bf7cf..c5e58c635 100644 --- a/milli/src/search/new/ranking_rule_graph/exactness/mod.rs +++ b/milli/src/search/new/ranking_rule_graph/exactness/mod.rs @@ -1,7 +1,7 @@ use roaring::RoaringBitmap; use super::{ComputedCondition, RankingRuleGraphTrait}; -use crate::score_details::{Rank, ScoreDetails}; +use crate::score_details::{self, Rank, ScoreDetails}; use crate::search::new::interner::{DedupInterner, Interned}; use crate::search::new::query_term::{ExactTerm, LocatedQueryTermSubset}; use crate::search::new::resolve_query_graph::compute_query_term_subset_docids; @@ -87,6 +87,6 @@ impl RankingRuleGraphTrait for ExactnessGraph { } fn rank_to_score(rank: Rank) -> ScoreDetails { - ScoreDetails::Exactness(rank) + ScoreDetails::ExactWords(score_details::ExactWords::from_rank(rank)) } } From 55cd7738b90889230879eff7cf3cb793176b5f7c Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 4 Jul 2023 15:54:08 +0200 Subject: [PATCH 13/54] Update snapshots --- ...sts__exactness__exactness_after_words.snap | 144 +++++++++--------- ...s__exactness_all_candidates_with_typo.snap | 40 ++--- ...ctness_attribute_starts_with_phrase-3.snap | 48 +++--- ...xactness_attribute_starts_with_phrase.snap | 32 ++-- ...xactness_attribute_starts_with_simple.snap | 24 +-- ...ollowed_by_typo_prefer_no_typo_prefix.snap | 40 ++--- ...__exactness__exactness_simple_ordered.snap | 72 ++++----- ...s__exactness__exactness_simple_random.snap | 48 +++--- ...xactness__exactness_simple_reversed-3.snap | 56 +++---- ..._exactness__exactness_simple_reversed.snap | 56 +++---- ...xactness__proximity_after_exactness-4.snap | 24 +-- ..._exactness__proximity_after_exactness.snap | 72 ++++----- ...exactness__typo_followed_by_exactness.snap | 32 ++-- ...sts__exactness__words_after_exactness.snap | 144 +++++++++--------- ...s__stop_words__stop_words_in_phrase-6.snap | 24 +-- milli/src/search/new/tests/stop_words.rs | 56 +++---- 16 files changed, 456 insertions(+), 456 deletions(-) diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap index ef95520bb..5463f0db5 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_after_words.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 9, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 9, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 8, + max_matching_words: 8, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 8, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 7, }, ), ], @@ -155,10 +155,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 7, }, ), ], @@ -175,10 +175,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -195,10 +195,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 5, }, ), ], @@ -215,10 +215,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -235,10 +235,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 4, }, ), ], @@ -255,10 +255,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -275,10 +275,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 3, }, ), ], @@ -295,10 +295,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -315,10 +315,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 2, }, ), ], @@ -335,10 +335,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -355,10 +355,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap index d48bf9933..bf3d9c403 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_all_candidates_with_typo.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 7, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 7, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 7, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 1, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 0, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap index 991ff4cee..d1b24fcec 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase-3.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 7, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap index 703f3cb7a..2e1e9fc56 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_phrase.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap index eb6141468..a7c0f4bc2 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_attribute_starts_with_simple.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap index 987e585a8..f2d94bf1a 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_followed_by_typo_prefer_no_typo_prefix.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Typo( @@ -41,10 +41,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), Typo( @@ -67,10 +67,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), Typo( @@ -93,10 +93,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), Typo( @@ -119,10 +119,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 4, }, ), Typo( diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap index c5993a09e..88b937145 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_ordered.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 9, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 8, + max_matching_words: 8, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -155,10 +155,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -175,10 +175,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap index d920eb4a0..07585bc2d 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_random.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap index d0bc0fb46..4fb863a3d 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed-3.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap index d0bc0fb46..4fb863a3d 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__exactness_simple_reversed.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap index 21c6da724..5db8f8da8 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness-4.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -41,10 +41,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -67,10 +67,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap index 7a33134cf..c99b50284 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__proximity_after_exactness.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -41,10 +41,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -67,10 +67,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), Proximity( @@ -93,10 +93,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -119,10 +119,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -145,10 +145,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -171,10 +171,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -197,10 +197,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( @@ -223,10 +223,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), Proximity( diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap index 6670f3e4f..320e1b878 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__typo_followed_by_exactness.snap @@ -21,10 +21,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -47,10 +47,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), ], @@ -73,10 +73,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 4, }, ), ], @@ -99,10 +99,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 4, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap index ef95520bb..5463f0db5 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__exactness__words_after_exactness.snap @@ -15,10 +15,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 10, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 9, + max_matching_words: 9, }, ), ], @@ -35,10 +35,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 7, - max_rank: 10, + ExactWords( + ExactWords { + matching_words: 6, + max_matching_words: 9, }, ), ], @@ -55,10 +55,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 9, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 8, + max_matching_words: 8, }, ), ], @@ -75,10 +75,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 9, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 8, }, ), ], @@ -95,10 +95,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -115,10 +115,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 8, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 7, + max_matching_words: 7, }, ), ], @@ -135,10 +135,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 7, }, ), ], @@ -155,10 +155,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 8, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 7, }, ), ], @@ -175,10 +175,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 6, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 5, + max_matching_words: 5, }, ), ], @@ -195,10 +195,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 6, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 5, }, ), ], @@ -215,10 +215,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 5, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 4, + max_matching_words: 4, }, ), ], @@ -235,10 +235,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 5, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 4, }, ), ], @@ -255,10 +255,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -275,10 +275,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 3, }, ), ], @@ -295,10 +295,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -315,10 +315,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 2, }, ), ], @@ -335,10 +335,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -355,10 +355,10 @@ expression: "format!(\"{document_ids_scores:#?}\")" ExactAttribute( ExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], diff --git a/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap b/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap index 1ca6a33a4..b12874c7b 100644 --- a/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap +++ b/milli/src/search/new/tests/snapshots/milli__search__new__tests__stop_words__stop_words_in_phrase-6.snap @@ -37,10 +37,10 @@ expression: "format!(\"{document_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -78,10 +78,10 @@ expression: "format!(\"{document_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], @@ -119,10 +119,10 @@ expression: "format!(\"{document_scores:#?}\")" ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], diff --git a/milli/src/search/new/tests/stop_words.rs b/milli/src/search/new/tests/stop_words.rs index 4ad587240..2b06264f5 100644 --- a/milli/src/search/new/tests/stop_words.rs +++ b/milli/src/search/new/tests/stop_words.rs @@ -120,10 +120,10 @@ fn test_ignore_stop_words() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -173,10 +173,10 @@ fn test_ignore_stop_words() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -226,10 +226,10 @@ fn test_ignore_stop_words() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -278,10 +278,10 @@ fn test_ignore_stop_words() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 3, - max_rank: 3, + ExactWords( + ExactWords { + matching_words: 2, + max_matching_words: 2, }, ), ], @@ -337,10 +337,10 @@ fn test_stop_words_in_phrase() { ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -378,10 +378,10 @@ fn test_stop_words_in_phrase() { ExactAttribute( MatchesStart, ), - Exactness( - Rank { - rank: 2, - max_rank: 2, + ExactWords( + ExactWords { + matching_words: 1, + max_matching_words: 1, }, ), ], @@ -430,10 +430,10 @@ fn test_stop_words_in_phrase() { ExactAttribute( NoExactMatch, ), - Exactness( - Rank { - rank: 4, - max_rank: 4, + ExactWords( + ExactWords { + matching_words: 3, + max_matching_words: 3, }, ), ], From da39a7b29ede569e25e39d8f9ec7acb7dc6f21e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 5 Jul 2023 17:27:51 +0200 Subject: [PATCH 14/54] Return the right analytics --- meilisearch/src/routes/indexes/settings.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/meilisearch/src/routes/indexes/settings.rs b/meilisearch/src/routes/indexes/settings.rs index fb5d8ff7a..1b48ae745 100644 --- a/meilisearch/src/routes/indexes/settings.rs +++ b/meilisearch/src/routes/indexes/settings.rs @@ -5,6 +5,7 @@ use index_scheduler::IndexScheduler; use log::debug; use meilisearch_types::deserr::DeserrJsonError; use meilisearch_types::error::ResponseError; +use meilisearch_types::facet_values_sort::FacetValuesSort; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::settings::{settings, RankingRuleView, Settings, Unchecked}; use meilisearch_types::tasks::KindWithContent; @@ -550,10 +551,16 @@ pub async fn update_all( .as_ref() .set() .and_then(|s| s.max_values_per_facet.as_ref().set()), - "sort_facet_values_by": new_settings.faceting + "sort_facet_values_by_star_count": new_settings.faceting .as_ref() .set() - .and_then(|s| s.sort_facet_values_by.as_ref().set()), + .and_then(|s| { + s.sort_facet_values_by.as_ref().set().map(|s| s.iter().any(|(k, v)| k == "*" && v == &FacetValuesSort::Count)) + }), + "sort_facet_values_by_total": new_settings.faceting + .as_ref() + .set() + .and_then(|s| s.sort_facet_values_by.as_ref().set().map(|s| s.len())), }, "pagination": { "max_total_hits": new_settings.pagination From d727ebee05906cb3b7138aef7ea722f8e2a22f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 5 Jul 2023 17:53:09 +0200 Subject: [PATCH 15/54] Fix the way we compute the 99th percentile --- meilisearch/src/analytics/segment_analytics.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 17170b582..30796dc88 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -761,17 +761,17 @@ impl SearchAggregator { if self.total_received == 0 { None } else { - // the index of the 99th percentage of value - let percentile_99th = 0.99 * (self.total_succeeded as f64 - 1.) + 1.; // we get all the values in a sorted manner let time_spent = self.time_spent.into_sorted_vec(); + // the index of the 99th percentage of value + let percentile_99th = time_spent.len() * 99 / 100; // We are only interested by the slowest value of the 99th fastest results - let time_spent = time_spent.get(percentile_99th as usize); + let time_spent = time_spent.get(percentile_99th); let properties = json!({ "user-agent": self.user_agents, "requests": { - "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), + "99th_response_time": time_spent.map(|t| format!("{:.2}", t)), "total_succeeded": self.total_succeeded, "total_failed": self.total_received.saturating_sub(self.total_succeeded), // just to be sure we never panics "total_received": self.total_received, From 11d024c61311616a97b81a52c0f4d9c95bbad8f1 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 6 Jul 2023 09:00:51 +0200 Subject: [PATCH 16/54] Authentication tests --- meilisearch/tests/auth/authorization.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meilisearch/tests/auth/authorization.rs b/meilisearch/tests/auth/authorization.rs index 58fba4481..5af214e5f 100644 --- a/meilisearch/tests/auth/authorization.rs +++ b/meilisearch/tests/auth/authorization.rs @@ -61,6 +61,8 @@ pub static AUTHORIZATIONS: Lazy hashset!{"keys.delete", "*"}, ("POST", "/keys") => hashset!{"keys.create", "*"}, ("GET", "/keys") => hashset!{"keys.get", "*"}, + ("GET", "/experimental-features") => hashset!{"experimental.get", "*"}, + ("PATCH", "/experimental-features") => hashset!{"experimental.update", "*"}, }; authorizations From 222615d3dfd73e890c9c5d353edd17a6c406c9a1 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 6 Jul 2023 09:01:05 +0200 Subject: [PATCH 17/54] Allow to get/set features in integration test server --- meilisearch/tests/common/server.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/meilisearch/tests/common/server.rs b/meilisearch/tests/common/server.rs index 66b82eace..40d8e8366 100644 --- a/meilisearch/tests/common/server.rs +++ b/meilisearch/tests/common/server.rs @@ -189,6 +189,14 @@ impl Server { let url = format!("/tasks/{}", update_id); self.service.get(url).await } + + pub async fn get_features(&self) -> (Value, StatusCode) { + self.service.get("/experimental-features").await + } + + pub async fn set_features(&self, value: Value) -> (Value, StatusCode) { + self.service.patch("/experimental-features", value).await + } } pub fn default_settings(dir: impl AsRef) -> Opt { From 76e1ee998831c1a6171eb33fd537dfd433ce813c Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 6 Jul 2023 09:01:28 +0200 Subject: [PATCH 18/54] integration test on "/experimental-features" route --- meilisearch/tests/features/mod.rs | 109 ++++++++++++++++++++++++++++++ meilisearch/tests/integration.rs | 1 + 2 files changed, 110 insertions(+) create mode 100644 meilisearch/tests/features/mod.rs diff --git a/meilisearch/tests/features/mod.rs b/meilisearch/tests/features/mod.rs new file mode 100644 index 000000000..045440f49 --- /dev/null +++ b/meilisearch/tests/features/mod.rs @@ -0,0 +1,109 @@ +use serde_json::json; + +use crate::common::Server; + +/// Feature name to test against. +/// This will have to be changed by a different one when that feature is stabilized. +/// All tests that need to set a feature can make use of this constant. +const FEATURE_NAME: &str = "vectorStore"; + +#[actix_rt::test] +async fn experimental_features() { + let server = Server::new().await; + + let (response, code) = server.get_features().await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": false + } + "###); + + let (response, code) = server.set_features(json!({FEATURE_NAME: true})).await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": true + } + "###); + + let (response, code) = server.get_features().await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": true + } + "###); + + // sending null does not change the value + let (response, code) = server.set_features(json!({FEATURE_NAME: null})).await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": true + } + "###); + + // not sending the field does not change the value + let (response, code) = server.set_features(json!({})).await; + + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "scoreDetails": false, + "vectorStore": true + } + "###); +} + +#[actix_rt::test] +async fn errors() { + let server = Server::new().await; + + // Sending a feature not in the list is an error + let (response, code) = server.set_features(json!({"NotAFeature": true})).await; + + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Unknown field `NotAFeature`: expected one of `scoreDetails`, `vectorStore`", + "code": "bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#bad_request" + } + "###); + + // The type must be a bool, not a number + let (response, code) = server.set_features(json!({FEATURE_NAME: 42})).await; + + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Invalid value type at `.vectorStore`: expected a boolean, but found a positive integer: `42`", + "code": "bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#bad_request" + } + "###); + + // The type must be a bool, not a string + let (response, code) = server.set_features(json!({FEATURE_NAME: "true"})).await; + + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Invalid value type at `.vectorStore`: expected a boolean, but found a string: `\"true\"`", + "code": "bad_request", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#bad_request" + } + "###); +} diff --git a/meilisearch/tests/integration.rs b/meilisearch/tests/integration.rs index 4383aea57..b6992791a 100644 --- a/meilisearch/tests/integration.rs +++ b/meilisearch/tests/integration.rs @@ -3,6 +3,7 @@ mod common; mod dashboard; mod documents; mod dumps; +mod features; mod index; mod search; mod settings; From 2d3cec11a74fc16bbce2430e6d6739ba2c664b6a Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 6 Jul 2023 09:02:02 +0200 Subject: [PATCH 19/54] Search integration test to check score details and vector store --- meilisearch/tests/search/mod.rs | 124 ++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/meilisearch/tests/search/mod.rs b/meilisearch/tests/search/mod.rs index 97556ae2a..e6eae7cb1 100644 --- a/meilisearch/tests/search/mod.rs +++ b/meilisearch/tests/search/mod.rs @@ -752,3 +752,127 @@ async fn faceting_max_values_per_facet() { ) .await; } + +#[actix_rt::test] +async fn experimental_feature_score_details() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + + index.add_documents(json!(documents), None).await; + index.wait_task(0).await; + + index + .search( + json!({ + "q": "train dragon", + "showRankingScoreDetails": true, + }), + |response, code| { + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Computing score details requires enabling the `score details` experimental feature. See https://github.com/meilisearch/product/discussions/674", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "###); + }, + ) + .await; + + let (response, code) = server.set_features(json!({"scoreDetails": true})).await; + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(response["scoreDetails"], @"true"); + + index + .search( + json!({ + "q": "train dragon", + "showRankingScoreDetails": true, + }), + |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "title": "How to Train Your Dragon: The Hidden World", + "id": "166428", + "_rankingScoreDetails": { + "words": { + "order": 0, + "matchingWords": 2, + "maxMatchingWords": 2, + "score": 1.0 + }, + "typo": { + "order": 1, + "typoCount": 0, + "maxTypoCount": 2, + "score": 1.0 + }, + "proximity": { + "order": 2, + "score": 0.875 + }, + "attribute": { + "order": 3, + "attribute_ranking_order_score": 1.0, + "query_word_distance_score": 0.8095238095238095, + "score": 0.9365079365079364 + }, + "exactness": { + "order": 4, + "matchType": "noExactMatch", + "matchingWords": 2, + "maxMatchingWords": 2, + "score": 0.3333333333333333 + } + } + } + ] + "###); + }, + ) + .await; +} + +#[actix_rt::test] +async fn experimental_feature_vector_store() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + + index.add_documents(json!(documents), None).await; + index.wait_task(0).await; + + let (response, code) = index + .search_post(json!({ + "vector": [1.0, 2.0, 3.0], + })) + .await; + meili_snap::snapshot!(code, @"400 Bad Request"); + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + { + "message": "Passing `vector` as a query parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "###); + + let (response, code) = server.set_features(json!({"vectorStore": true})).await; + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(response["vectorStore"], @"true"); + + let (response, code) = index + .search_post(json!({ + "vector": [1.0, 2.0, 3.0], + })) + .await; + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @"[]"); +} From 86b834c9e428c581a676aaa7c3ee983a28a0b387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 5 Jul 2023 10:58:10 +0200 Subject: [PATCH 20/54] Display the total number of tasks in the tasks route --- index-scheduler/src/lib.rs | 42 +++++++++++++++++++++++---------- meilisearch/src/routes/tasks.rs | 29 ++++++++++------------- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 1e7ba6e21..99b17b686 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -138,6 +138,12 @@ impl Query { index_vec.push(index_uid); Self { index_uids: Some(index_vec), ..self } } + + // Removes the `from` and `limit` restrictions from the query. + // Useful to get the total number of tasks matching a filter. + pub fn without_limits(self) -> Self { + Query { limit: None, from: None, ..self } + } } #[derive(Debug, Clone)] @@ -822,7 +828,8 @@ impl IndexScheduler { Ok(nbr_index_processing_tasks > 0) } - /// Return the task ids matching the query from the user's point of view. + /// Return the task ids matching the query along with the total number of tasks + /// by ignoring the from and limit parameters from the user's point of view. /// /// There are two differences between an internal query and a query executed by /// the user. @@ -835,7 +842,13 @@ impl IndexScheduler { rtxn: &RoTxn, query: &Query, filters: &meilisearch_auth::AuthFilter, - ) -> Result { + ) -> Result<(RoaringBitmap, u64)> { + // compute all tasks matching the filter by ignoring the limits, to find the number of tasks matching + // the filter. + // As this causes us to compute the filter twice it is slightly inefficient, but doing it this way spares + // us from modifying the underlying implementation, and the performance remains sufficient. + // Should this change, we would modify `get_task_ids` to directly return the number of matching tasks. + let total_tasks = self.get_task_ids(rtxn, &query.clone().without_limits())?; let mut tasks = self.get_task_ids(rtxn, query)?; // If the query contains a list of index uid or there is a finite list of authorized indexes, @@ -858,10 +871,11 @@ impl IndexScheduler { } } - Ok(tasks) + Ok((tasks, total_tasks.len())) } - /// Return the tasks matching the query from the user's point of view. + /// Return the tasks matching the query from the user's point of view along + /// with the total number of tasks matching the query, ignoring from and limit. /// /// There are two differences between an internal query and a query executed by /// the user. @@ -873,11 +887,10 @@ impl IndexScheduler { &self, query: Query, filters: &meilisearch_auth::AuthFilter, - ) -> Result> { + ) -> Result<(Vec, u64)> { let rtxn = self.env.read_txn()?; - let tasks = self.get_task_ids_from_authorized_indexes(&rtxn, &query, filters)?; - + let (tasks, total) = self.get_task_ids_from_authorized_indexes(&rtxn, &query, filters)?; let tasks = self.get_existing_tasks( &rtxn, tasks.into_iter().rev().take(query.limit.unwrap_or(u32::MAX) as usize), @@ -888,16 +901,19 @@ impl IndexScheduler { let ret = tasks.into_iter(); if processing.is_empty() { - Ok(ret.collect()) + Ok((ret.collect(), total)) } else { - Ok(ret - .map(|task| match processing.contains(task.uid) { - true => { + Ok(( + ret.map(|task| { + if processing.contains(task.uid) { Task { status: Status::Processing, started_at: Some(started_at), ..task } + } else { + task } - false => task, }) - .collect()) + .collect(), + total, + )) } } diff --git a/meilisearch/src/routes/tasks.rs b/meilisearch/src/routes/tasks.rs index 2713d0988..f7d4c44d7 100644 --- a/meilisearch/src/routes/tasks.rs +++ b/meilisearch/src/routes/tasks.rs @@ -325,7 +325,7 @@ async fn cancel_tasks( let query = params.into_query(); - let tasks = index_scheduler.get_task_ids_from_authorized_indexes( + let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes( &index_scheduler.read_txn()?, &query, index_scheduler.filters(), @@ -370,7 +370,7 @@ async fn delete_tasks( ); let query = params.into_query(); - let tasks = index_scheduler.get_task_ids_from_authorized_indexes( + let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes( &index_scheduler.read_txn()?, &query, index_scheduler.filters(), @@ -387,6 +387,7 @@ async fn delete_tasks( #[derive(Debug, Serialize)] pub struct AllTasks { results: Vec, + total: u64, limit: u32, from: Option, next: Option, @@ -406,23 +407,17 @@ async fn get_tasks( let limit = params.limit.0; let query = params.into_query(); - let mut tasks_results: Vec = index_scheduler - .get_tasks_from_authorized_indexes(query, index_scheduler.filters())? - .into_iter() - .map(|t| TaskView::from_task(&t)) - .collect(); + let filters = index_scheduler.filters(); + let (tasks, total) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?; + let mut results: Vec<_> = tasks.iter().map(TaskView::from_task).collect(); // If we were able to fetch the number +1 tasks we asked // it means that there is more to come. - let next = if tasks_results.len() == limit as usize { - tasks_results.pop().map(|t| t.uid) - } else { - None - }; + let next = if results.len() == limit as usize { results.pop().map(|t| t.uid) } else { None }; - let from = tasks_results.first().map(|t| t.uid); + let from = results.first().map(|t| t.uid); + let tasks = AllTasks { results, limit: limit.saturating_sub(1), total, from, next }; - let tasks = AllTasks { results: tasks_results, limit: limit.saturating_sub(1), from, next }; Ok(HttpResponse::Ok().json(tasks)) } @@ -444,10 +439,10 @@ async fn get_task( analytics.publish("Tasks Seen".to_string(), json!({ "per_task_uid": true }), Some(&req)); let query = index_scheduler::Query { uids: Some(vec![task_uid]), ..Query::default() }; + let filters = index_scheduler.filters(); + let (tasks, _) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?; - if let Some(task) = - index_scheduler.get_tasks_from_authorized_indexes(query, index_scheduler.filters())?.first() - { + if let Some(task) = tasks.first() { let task_view = TaskView::from_task(task); Ok(HttpResponse::Ok().json(task_view)) } else { From 22762808ab2327aab76e37c521b42b7f718cbd18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 5 Jul 2023 11:00:40 +0200 Subject: [PATCH 21/54] Fix the tests --- index-scheduler/src/lib.rs | 60 +++++++++---------- meilisearch/tests/dumps/mod.rs | 16 ++--- .../mod.rs/import_dump_v1_movie_raw/1.snap | 1 + .../mod.rs/import_dump_v1_movie_raw/2.snap | 1 + .../mod.rs/import_dump_v1_movie_raw/3.snap | 1 + .../mod.rs/import_dump_v1_movie_raw/4.snap | 1 + .../mod.rs/import_dump_v1_movie_raw/5.snap | 1 + .../mod.rs/import_dump_v1_movie_raw/6.snap | 1 + .../mod.rs/import_dump_v1_movie_raw/7.snap | 1 + .../import_dump_v1_movie_with_settings/1.snap | 1 + .../import_dump_v1_movie_with_settings/2.snap | 1 + .../import_dump_v1_movie_with_settings/3.snap | 1 + .../import_dump_v1_movie_with_settings/4.snap | 1 + .../import_dump_v1_movie_with_settings/5.snap | 1 + .../import_dump_v1_movie_with_settings/6.snap | 1 + .../import_dump_v1_movie_with_settings/7.snap | 1 + .../1.snap | 1 + .../2.snap | 1 + .../3.snap | 1 + .../4.snap | 1 + .../5.snap | 1 + .../6.snap | 1 + .../7.snap | 1 + .../mod.rs/import_dump_v2_movie_raw/1.snap | 1 + .../mod.rs/import_dump_v2_movie_raw/2.snap | 1 + .../mod.rs/import_dump_v2_movie_raw/3.snap | 1 + .../mod.rs/import_dump_v2_movie_raw/4.snap | 1 + .../mod.rs/import_dump_v2_movie_raw/5.snap | 1 + .../mod.rs/import_dump_v2_movie_raw/6.snap | 1 + .../mod.rs/import_dump_v2_movie_raw/7.snap | 1 + .../import_dump_v2_movie_with_settings/1.snap | 1 + .../import_dump_v2_movie_with_settings/2.snap | 1 + .../import_dump_v2_movie_with_settings/3.snap | 1 + .../import_dump_v2_movie_with_settings/4.snap | 1 + .../import_dump_v2_movie_with_settings/5.snap | 1 + .../import_dump_v2_movie_with_settings/6.snap | 1 + .../import_dump_v2_movie_with_settings/7.snap | 1 + .../1.snap | 1 + .../2.snap | 1 + .../3.snap | 1 + .../4.snap | 1 + .../5.snap | 1 + .../6.snap | 1 + .../7.snap | 1 + .../mod.rs/import_dump_v3_movie_raw/1.snap | 1 + .../mod.rs/import_dump_v3_movie_raw/2.snap | 1 + .../mod.rs/import_dump_v3_movie_raw/3.snap | 1 + .../mod.rs/import_dump_v3_movie_raw/4.snap | 1 + .../mod.rs/import_dump_v3_movie_raw/5.snap | 1 + .../mod.rs/import_dump_v3_movie_raw/6.snap | 1 + .../mod.rs/import_dump_v3_movie_raw/7.snap | 1 + .../import_dump_v3_movie_with_settings/1.snap | 1 + .../import_dump_v3_movie_with_settings/2.snap | 1 + .../import_dump_v3_movie_with_settings/3.snap | 1 + .../import_dump_v3_movie_with_settings/4.snap | 1 + .../import_dump_v3_movie_with_settings/5.snap | 1 + .../import_dump_v3_movie_with_settings/6.snap | 1 + .../import_dump_v3_movie_with_settings/7.snap | 1 + .../1.snap | 1 + .../2.snap | 1 + .../3.snap | 1 + .../4.snap | 1 + .../5.snap | 1 + .../6.snap | 1 + .../7.snap | 1 + .../mod.rs/import_dump_v4_movie_raw/1.snap | 1 + .../mod.rs/import_dump_v4_movie_raw/2.snap | 1 + .../mod.rs/import_dump_v4_movie_raw/3.snap | 1 + .../mod.rs/import_dump_v4_movie_raw/4.snap | 1 + .../mod.rs/import_dump_v4_movie_raw/5.snap | 1 + .../mod.rs/import_dump_v4_movie_raw/6.snap | 1 + .../mod.rs/import_dump_v4_movie_raw/7.snap | 1 + .../import_dump_v4_movie_with_settings/1.snap | 1 + .../import_dump_v4_movie_with_settings/2.snap | 1 + .../import_dump_v4_movie_with_settings/3.snap | 1 + .../import_dump_v4_movie_with_settings/4.snap | 1 + .../import_dump_v4_movie_with_settings/5.snap | 1 + .../import_dump_v4_movie_with_settings/6.snap | 1 + .../import_dump_v4_movie_with_settings/7.snap | 1 + .../1.snap | 1 + .../2.snap | 1 + .../3.snap | 1 + .../4.snap | 1 + .../5.snap | 1 + .../6.snap | 1 + .../7.snap | 1 + .../snapshots/mod.rs/import_dump_v5/1.snap | 1 + .../snapshots/mod.rs/import_dump_v5/2.snap | 1 + .../snapshots/mod.rs/import_dump_v5/3.snap | 1 + .../snapshots/mod.rs/import_dump_v5/4.snap | 1 + .../snapshots/mod.rs/import_dump_v5/5.snap | 1 + .../snapshots/mod.rs/import_dump_v5/6.snap | 1 + .../snapshots/mod.rs/import_dump_v5/7.snap | 1 + meilisearch/tests/swap_indexes/mod.rs | 4 ++ 94 files changed, 133 insertions(+), 38 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 99b17b686..9b921a756 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -2797,43 +2797,43 @@ mod tests { let rtxn = index_scheduler.env.read_txn().unwrap(); let query = Query { limit: Some(0), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[]"); let query = Query { limit: Some(1), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[2,]"); let query = Query { limit: Some(2), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[1,2,]"); let query = Query { from: Some(1), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,]"); let query = Query { from: Some(2), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,2,]"); let query = Query { from: Some(1), limit: Some(1), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[1,]"); let query = Query { from: Some(1), limit: Some(2), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,]"); @@ -2860,13 +2860,13 @@ mod tests { let rtxn = index_scheduler.env.read_txn().unwrap(); let query = Query { statuses: Some(vec![Status::Processing]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,]"); // only the processing tasks in the first tick let query = Query { statuses: Some(vec![Status::Enqueued]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[1,2,]"); // only the enqueued tasks in the first tick @@ -2875,7 +2875,7 @@ mod tests { statuses: Some(vec![Status::Enqueued, Status::Processing]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); snapshot!(snapshot_bitmap(&tasks), @"[0,1,2,]"); // both enqueued and processing tasks in the first tick @@ -2885,7 +2885,7 @@ mod tests { after_started_at: Some(start_time), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both enqueued and processing tasks in the first tick, but limited to those with a started_at @@ -2897,7 +2897,7 @@ mod tests { before_started_at: Some(start_time), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both enqueued and processing tasks in the first tick, but limited to those with a started_at @@ -2910,7 +2910,7 @@ mod tests { before_started_at: Some(start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both enqueued and processing tasks in the first tick, but limited to those with a started_at @@ -2937,7 +2937,7 @@ mod tests { before_started_at: Some(start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both succeeded and processing tasks in the first tick, but limited to those with a started_at @@ -2950,7 +2950,7 @@ mod tests { before_started_at: Some(start_time), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both succeeded and processing tasks in the first tick, but limited to those with a started_at @@ -2963,7 +2963,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // both succeeded and processing tasks in the first tick, but limited to those with a started_at @@ -2983,7 +2983,7 @@ mod tests { let rtxn = index_scheduler.env.read_txn().unwrap(); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // we run the same query to verify that, and indeed find that the last task is matched @@ -2995,7 +2995,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // enqueued, succeeded, or processing tasks started after the second part of the test, should @@ -3007,7 +3007,7 @@ mod tests { // now the last task should have failed snapshot!(snapshot_index_scheduler(&index_scheduler), name: "end"); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // so running the last query should return nothing @@ -3019,7 +3019,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // but the same query on failed tasks should return the last task @@ -3031,7 +3031,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // but the same query on failed tasks should return the last task @@ -3044,7 +3044,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // same query but with an invalid uid @@ -3057,7 +3057,7 @@ mod tests { before_started_at: Some(second_start_time + Duration::minutes(1)), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // same query but with a valid uid @@ -3089,14 +3089,14 @@ mod tests { let rtxn = index_scheduler.env.read_txn().unwrap(); let query = Query { index_uids: Some(vec!["catto".to_owned()]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // only the first task associated with catto is returned, the indexSwap tasks are excluded! snapshot!(snapshot_bitmap(&tasks), @"[0,]"); let query = Query { index_uids: Some(vec!["catto".to_owned()]), ..Default::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes( &rtxn, &query, @@ -3110,7 +3110,7 @@ mod tests { snapshot!(snapshot_bitmap(&tasks), @"[]"); let query = Query::default(); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes( &rtxn, &query, @@ -3124,7 +3124,7 @@ mod tests { snapshot!(snapshot_bitmap(&tasks), @"[1,]"); let query = Query::default(); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes( &rtxn, &query, @@ -3143,7 +3143,7 @@ mod tests { snapshot!(snapshot_bitmap(&tasks), @"[0,1,]"); let query = Query::default(); - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // we asked for all the tasks with all index authorized -> all tasks returned @@ -3176,7 +3176,7 @@ mod tests { let rtxn = index_scheduler.read_txn().unwrap(); let query = Query { canceled_by: Some(vec![task_cancelation.uid]), ..Query::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default()) .unwrap(); // 0 is not returned because it was not canceled, 3 is not returned because it is the uid of the @@ -3184,7 +3184,7 @@ mod tests { snapshot!(snapshot_bitmap(&tasks), @"[1,2,]"); let query = Query { canceled_by: Some(vec![task_cancelation.uid]), ..Query::default() }; - let tasks = index_scheduler + let (tasks, _) = index_scheduler .get_task_ids_from_authorized_indexes( &rtxn, &query, diff --git a/meilisearch/tests/dumps/mod.rs b/meilisearch/tests/dumps/mod.rs index ce225cdf7..f041867a8 100644 --- a/meilisearch/tests/dumps/mod.rs +++ b/meilisearch/tests/dumps/mod.rs @@ -43,7 +43,7 @@ async fn import_dump_v1_movie_raw() { assert_eq!(code, 200); assert_eq!( tasks, - json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.317060500S", "enqueuedAt": "2021-09-08T09:08:45.153219Z", "startedAt": "2021-09-08T09:08:45.3961665Z", "finishedAt": "2021-09-08T09:08:54.713227Z" }], "limit": 20, "from": 0, "next": null }) + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.317060500S", "enqueuedAt": "2021-09-08T09:08:45.153219Z", "startedAt": "2021-09-08T09:08:45.3961665Z", "finishedAt": "2021-09-08T09:08:54.713227Z" }], "total": 1, "limit": 20, "from": 0, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -135,7 +135,7 @@ async fn import_dump_v1_movie_with_settings() { assert_eq!(code, 200); assert_eq!( tasks, - json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["genres", "id", "overview", "poster", "release_date", "title"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT7.288826907S", "enqueuedAt": "2021-09-08T09:34:40.882977Z", "startedAt": "2021-09-08T09:34:40.883073093Z", "finishedAt": "2021-09-08T09:34:48.1719Z"}, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.090735774S", "enqueuedAt": "2021-09-08T09:34:16.036101Z", "startedAt": "2021-09-08T09:34:16.261191226Z", "finishedAt": "2021-09-08T09:34:25.351927Z" }], "limit": 20, "from": 1, "next": null }) + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["genres", "id", "overview", "poster", "release_date", "title"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT7.288826907S", "enqueuedAt": "2021-09-08T09:34:40.882977Z", "startedAt": "2021-09-08T09:34:40.883073093Z", "finishedAt": "2021-09-08T09:34:48.1719Z"}, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.090735774S", "enqueuedAt": "2021-09-08T09:34:16.036101Z", "startedAt": "2021-09-08T09:34:16.261191226Z", "finishedAt": "2021-09-08T09:34:25.351927Z" }], "total": 2, "limit": 20, "from": 1, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -317,7 +317,7 @@ async fn import_dump_v2_movie_raw() { assert_eq!(code, 200); assert_eq!( tasks, - json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "limit": 20, "from": 0, "next": null }) + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "total": 1, "limit": 20, "from": 0, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -409,7 +409,7 @@ async fn import_dump_v2_movie_with_settings() { assert_eq!(code, 200); assert_eq!( tasks, - json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "limit": 20, "from": 1, "next": null }) + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "total": 2, "limit": 20, "from": 1, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -591,7 +591,7 @@ async fn import_dump_v3_movie_raw() { assert_eq!(code, 200); assert_eq!( tasks, - json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "limit": 20, "from": 0, "next": null }) + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "total": 1, "limit": 20, "from": 0, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -683,7 +683,7 @@ async fn import_dump_v3_movie_with_settings() { assert_eq!(code, 200); assert_eq!( tasks, - json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "limit": 20, "from": 1, "next": null }) + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "total": 2, "limit": 20, "from": 1, "next": null }) ); // finally we're just going to check that we can["results"] still get a few documents by id @@ -865,7 +865,7 @@ async fn import_dump_v4_movie_raw() { assert_eq!(code, 200); assert_eq!( tasks, - json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "limit" : 20, "from": 0, "next": null }) + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT41.751156S", "enqueuedAt": "2021-09-08T08:30:30.550282Z", "startedAt": "2021-09-08T08:30:30.553012Z", "finishedAt": "2021-09-08T08:31:12.304168Z" }], "total": 1, "limit" : 20, "from": 0, "next": null }) ); // finally we're just going to check that we can still get a few documents by id @@ -957,7 +957,7 @@ async fn import_dump_v4_movie_with_settings() { assert_eq!(code, 200); assert_eq!( tasks, - json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "limit": 20, "from": 1, "next": null }) + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["title", "genres", "overview", "poster", "release_date"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT37.488777S", "enqueuedAt": "2021-09-08T08:24:02.323444Z", "startedAt": "2021-09-08T08:24:02.324145Z", "finishedAt": "2021-09-08T08:24:39.812922Z" }, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31944 }, "error": null, "duration": "PT39.941318S", "enqueuedAt": "2021-09-08T08:21:14.742672Z", "startedAt": "2021-09-08T08:21:14.750166Z", "finishedAt": "2021-09-08T08:21:54.691484Z" }], "total": 2, "limit": 20, "from": 1, "next": null }) ); // finally we're just going to check that we can still get a few documents by id diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap index f062ee002..7227fd4cc 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_raw/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:08:54.713227Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap index 986b71132..d22c13b72 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:25.351927Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap index 986b71132..d22c13b72 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:25.351927Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/3.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/4.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/5.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/6.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap index e2e059fac..b49f68a59 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_movie_with_settings/7.snap @@ -40,6 +40,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:34:48.1719Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap index 84a9c4fd5..beb25ce95 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/1.snap @@ -45,6 +45,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:26:57.319083Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap index fa8a9305b..add41439f 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 92, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap index fa8a9305b..c9e368061 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v1_rubygems_with_settings/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T09:28:46.369971Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_raw/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/3.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/4.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/5.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/6.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_movie_with_settings/7.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap index 00d18a4f0..92613ce34 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/1.snap @@ -41,6 +41,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:40:28.669652Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap index 6a2fbe74f..bfef698e1 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 92, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v2_rubygems_with_settings/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_raw/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/3.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/4.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/5.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/6.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_movie_with_settings/7.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap index 00d18a4f0..92613ce34 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/1.snap @@ -41,6 +41,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:40:28.669652Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap index 6a2fbe74f..bfef698e1 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 92, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v3_rubygems_with_settings/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap index 6fb6c0e80..33f1b46c5 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_raw/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:31:12.304168Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap index 17dc6a4ee..6acf1f564 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:21:54.691484Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/3.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/4.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/5.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/6.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap index 4253f1d0e..22fad9f58 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_movie_with_settings/7.snap @@ -36,6 +36,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:24:39.812922Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap index 00d18a4f0..92613ce34 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/1.snap @@ -41,6 +41,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:40:28.669652Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap index 6a2fbe74f..bfef698e1 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 92, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/3.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/4.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/5.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/6.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap index 6a2fbe74f..3fa18f512 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v4_rubygems_with_settings/7.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2021-09-08T08:51:53.095314Z" } ], + "total": 93, "limit": 1, "from": 92, "next": 91 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap index 4ca8cbba0..b8c0d07b1 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/1.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2022-06-08T14:59:24.804443Z" } ], + "total": 1, "limit": 1, "from": 0, "next": null diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap index a5cee8ace..b31fb744f 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/2.snap @@ -20,6 +20,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "2022-06-08T14:59:29.997781Z" } ], + "total": 2, "limit": 1, "from": 1, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap index 29b42313b..082cc97ee 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/3.snap @@ -19,6 +19,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 5, "limit": 1, "from": 4, "next": 3 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap index 9e63e3340..54e8f84bf 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/4.snap @@ -24,6 +24,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 2, "limit": 1, "from": 2, "next": 0 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap index 29b42313b..082cc97ee 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/5.snap @@ -19,6 +19,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 5, "limit": 1, "from": 4, "next": 3 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap index 29b42313b..082cc97ee 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/6.snap @@ -19,6 +19,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 5, "limit": 1, "from": 4, "next": 3 diff --git a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap index 29b42313b..082cc97ee 100644 --- a/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap +++ b/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v5/7.snap @@ -19,6 +19,7 @@ source: meilisearch/tests/dumps/mod.rs "finishedAt": "[date]" } ], + "total": 5, "limit": 1, "from": 4, "next": 3 diff --git a/meilisearch/tests/swap_indexes/mod.rs b/meilisearch/tests/swap_indexes/mod.rs index 42d92e2af..4d01995b5 100644 --- a/meilisearch/tests/swap_indexes/mod.rs +++ b/meilisearch/tests/swap_indexes/mod.rs @@ -55,6 +55,7 @@ async fn swap_indexes() { "finishedAt": "[date]" } ], + "total": 2, "limit": 20, "from": 1, "next": null @@ -128,6 +129,7 @@ async fn swap_indexes() { "finishedAt": "[date]" } ], + "total": 3, "limit": 20, "from": 2, "next": null @@ -193,6 +195,7 @@ async fn swap_indexes() { "finishedAt": "[date]" } ], + "total": 5, "limit": 2, "from": 4, "next": 2 @@ -336,6 +339,7 @@ async fn swap_indexes() { "finishedAt": "[date]" } ], + "total": 6, "limit": 20, "from": 5, "next": null From da02a9cf32366cd5dfa9a77e149fb30481fed6f4 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 6 Jul 2023 14:20:58 +0200 Subject: [PATCH 22/54] Make RuntimeTogglableFeatures Eq --- meilisearch-types/src/features.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-types/src/features.rs b/meilisearch-types/src/features.rs index 6d02fc47b..f62300485 100644 --- a/meilisearch-types/src/features.rs +++ b/meilisearch-types/src/features.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] #[serde(rename_all = "camelCase", default)] pub struct RuntimeTogglableFeatures { pub score_details: bool, From dd6519b64fbc7d8aa297f17f04f0ee76efc46122 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 6 Jul 2023 14:22:29 +0200 Subject: [PATCH 23/54] Dump tests --- dump/src/lib.rs | 14 +++++- dump/src/reader/mod.rs | 49 +++++++++++++++++++- dump/src/writer.rs | 1 + dump/tests/assets/v6-with-experimental.dump | Bin 0 -> 1297 bytes 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 dump/tests/assets/v6-with-experimental.dump diff --git a/dump/src/lib.rs b/dump/src/lib.rs index 1e21eed05..33bd8c63b 100644 --- a/dump/src/lib.rs +++ b/dump/src/lib.rs @@ -210,6 +210,7 @@ pub(crate) mod test { use big_s::S; use maplit::{btreemap, btreeset}; use meilisearch_types::facet_values_sort::FacetValuesSort; + use meilisearch_types::features::RuntimeTogglableFeatures; use meilisearch_types::index_uid_pattern::IndexUidPattern; use meilisearch_types::keys::{Action, Key}; use meilisearch_types::milli; @@ -418,7 +419,10 @@ pub(crate) mod test { } keys.flush().unwrap(); - // ========== TODO: create features here + // ========== experimental features + let features = create_test_features(); + + dump.create_experimental_features(features).unwrap(); // create the dump let mut file = tempfile::tempfile().unwrap(); @@ -428,6 +432,10 @@ pub(crate) mod test { file } + fn create_test_features() -> RuntimeTogglableFeatures { + RuntimeTogglableFeatures { vector_store: true, ..Default::default() } + } + #[test] fn test_creating_and_read_dump() { let mut file = create_test_dump(); @@ -472,5 +480,9 @@ pub(crate) mod test { for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) { assert_eq!(key.unwrap(), expected); } + + // ==== checking the features + let expected = create_test_features(); + assert_eq!(dump.features().unwrap().unwrap(), expected); } } diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index 0899579f8..af02888d2 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -195,8 +195,53 @@ pub(crate) mod test { use meili_snap::insta; use super::*; + use crate::reader::v6::RuntimeTogglableFeatures; - // TODO: add `features` to tests + #[test] + fn import_dump_v6_experimental() { + let dump = File::open("tests/assets/v6-with-experimental.dump").unwrap(); + let mut dump = DumpReader::open(dump).unwrap(); + + // top level infos + insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00"); + insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None"); + + // tasks + let tasks = dump.tasks().unwrap().collect::>>().unwrap(); + let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"d45cd8571703e58ae53c7bd7ce3f5c22"); + assert_eq!(update_files.len(), 2); + assert!(update_files[0].is_none()); // the dump creation + assert!(update_files[1].is_none()); // the processed document addition + + // keys + let keys = dump.keys().unwrap().collect::>>().unwrap(); + meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"13c2da155e9729c2344688cab29af71d"); + + // indexes + let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); + // the index are not ordered in any way by default + indexes.sort_by_key(|index| index.metadata().uid.to_string()); + + let mut test = indexes.pop().unwrap(); + assert!(indexes.is_empty()); + + insta::assert_json_snapshot!(test.metadata(), @r###" + { + "uid": "test", + "primaryKey": "id", + "createdAt": "2023-07-06T07:07:41.364694Z", + "updatedAt": "2023-07-06T07:07:41.396114Z" + } + "###); + + assert_eq!(test.documents().unwrap().count(), 1); + + assert_eq!( + dump.features().unwrap().unwrap(), + RuntimeTogglableFeatures { vector_store: true, ..Default::default() } + ); + } #[test] fn import_dump_v5() { @@ -274,6 +319,8 @@ pub(crate) mod test { let documents = spells.documents().unwrap().collect::>>().unwrap(); assert_eq!(documents.len(), 10); meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce"); + + assert_eq!(dump.features().unwrap(), None); } #[test] diff --git a/dump/src/writer.rs b/dump/src/writer.rs index 695610f78..3c8126876 100644 --- a/dump/src/writer.rs +++ b/dump/src/writer.rs @@ -292,6 +292,7 @@ pub(crate) mod test { │ ├---- update_files/ │ │ └---- 1.jsonl │ └---- queue.jsonl + ├---- experimental-features.json ├---- instance_uid.uuid ├---- keys.jsonl └---- metadata.json diff --git a/dump/tests/assets/v6-with-experimental.dump b/dump/tests/assets/v6-with-experimental.dump new file mode 100644 index 0000000000000000000000000000000000000000..ac6833dc3b21f3036a9752c8dfbdc37d1f903c55 GIT binary patch literal 1297 zcmV+s1@8JEiwFP!00000|Ls~`liD^E7Fvt2yd#%Ew`g@{w<0-wTvU5s&d;C2&3fwmS4WNqu znW9p4PJ}1K-;3P1jei5!j{jLIl5#=0lKnrW;92LH0O{;T7(W_+F9;sSAIFgYjSS)s z9{&|$DH?fQ#L0r95mGZI9rzGO&j=?i7rCQ^lgwx`chrn1hu-J3V0xO6%$ZW6%7RMg zP84&8z^PnOm{KWPzv{I3{;`w)3#v#;6fx=Fc{Uu){|N8pe-OlN{x^UXD)j>Z>G-2A z$|18!vHX=Tp$!gnQBpvP)AU?HgYWu5&y9O-bmhh)_+YOeL^vAaPheT*>0!%Z<4ZBH-HsNnane?)QPJKHYs5ypwaK>HTt8ABqc>M zBa@7N<=~WMiWYkZ^=YC|Uu!Q2pTqq6SZ3A@kcL#klS3_YVIDr>o-LSK!ak`_Mb;f` z=sigkr&5|LQzHM(oNjwua>17i__bPh^)10XPSmDxAn zP!%QZq6OpH=_UJ{z7@q4SZj|+aH{vwCD#DonYGM zg52MbtfcY-E#B&OU}3InZ`S;_M}J;wjdG3schg zaM$1PgxzLyy_(H6od0g0=!J9Und}#pQ-gk8ym|YP4Dt$oi zJ$K~Cecu~~gO=OBILzttnbp319Layr+voq#)UN-V0OUWFNx^bUC(*T}4pUn4v)tdA z7GeRyAJscX+d~_z@PbZBnJMSeTAqJ+=V;-eTS*G(B%UP6!0W{W6811ANpIqjxEF(D zkobdi=n>8Cfz(6SW<6lL)bzeumTuM>KvNkI(=kysm9KOO)ACSm2uD8d!)zK3hFikK zantpyM}}i63Wx9|h@<@0s-Pf|a-l#bt%gYejW(3L@4O+U-ucP75XJ_sU064Q!wGpg z$yjn=IL}j9hU|@IM1l}_an!?PfO|Ohg5F^2CA}~pbT}NwNsK4Eqxt&T1~Lree>slE zvXy>c4iu5|(^h|`b|Er`WJsAx= z!vFjL@995&7`6IO6R7GxZp9R?w~IkAGO7gXfMl5@l|N)geF-g?#AXy#)g*|7H0G(EwqxZ$Nd4%a>1x+aX3_NAigYj*d&~B^tO1YW+hbmmK9PvNd0V$5u!u%lA zmQ%)=oE-qUGBVj()^K#edK!VlUQ5<>k;RM^LOuio+NwCV)ciW{%ixw?fJdS?=7OL?- H05AXmAUBim literal 0 HcmV?d00001 From bb40ce6e35b5e6a13eaf7600b32d2e94e55cb2d7 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 10 Jul 2023 08:48:04 +0200 Subject: [PATCH 24/54] Experimental features analytics match the spec --- meilisearch/src/routes/features.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/meilisearch/src/routes/features.rs b/meilisearch/src/routes/features.rs index de5becfd1..a2822b4d4 100644 --- a/meilisearch/src/routes/features.rs +++ b/meilisearch/src/routes/features.rs @@ -64,7 +64,20 @@ async fn patch_features( vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store), }; - analytics.publish("Experimental features Updated".to_string(), json!(new_features), Some(&req)); + // explicitly destructure for analytics rather than using the `Serialize` implementation, because + // the it renames to camelCase, which we don't want for analytics. + // **Do not** ignore fields with `..` or `_` here, because we want to add them in the future. + let meilisearch_types::features::RuntimeTogglableFeatures { score_details, vector_store } = + new_features; + + analytics.publish( + "Experimental features Updated".to_string(), + json!({ + "score_details": score_details, + "vector_store": vector_store, + }), + Some(&req), + ); index_scheduler.put_runtime_features(new_features)?; Ok(HttpResponse::Ok().json(new_features)) } From 40fa59d64ca5bc105cc79dc9b884b9fd45ac6268 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 10 Jul 2023 09:26:59 +0200 Subject: [PATCH 25/54] Sort by lexicographic order after normalization --- milli/src/update/settings.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/milli/src/update/settings.rs b/milli/src/update/settings.rs index 5b8e5a21c..33f86a4bb 100644 --- a/milli/src/update/settings.rs +++ b/milli/src/update/settings.rs @@ -425,13 +425,14 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { let current = self.index.stop_words(self.wtxn)?; // Apply an unlossy normalization on stop_words - let stop_words = stop_words + let stop_words: BTreeSet = stop_words .iter() - .map(|w| w.as_str().normalize(&Default::default()).into_owned()); + .map(|w| w.as_str().normalize(&Default::default()).into_owned()) + .collect(); // since we can't compare a BTreeSet with an FST we are going to convert the // BTreeSet to an FST and then compare bytes per bytes the two FSTs. - let fst = fst::Set::from_iter(stop_words)?; + let fst = fst::Set::from_iter(stop_words.into_iter())?; // Does the new FST differ from the previous one? if current From 106f98aa72b3e793f43dd9462078b0b57f50e682 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 10 Jul 2023 11:45:43 +0200 Subject: [PATCH 26/54] Add "scoring.*" analytics to multi search route --- meilisearch/src/analytics/segment_analytics.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 30796dc88..1a8843a37 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -843,6 +843,10 @@ pub struct MultiSearchAggregator { // sum of the number of search queries in the requests, use with total_received to compute an average total_search_count: usize, + // scoring + show_ranking_score: bool, + show_ranking_score_details: bool, + // context user_agents: HashSet, } @@ -856,6 +860,9 @@ impl MultiSearchAggregator { let distinct_indexes: HashSet<_> = query.iter().map(|query| query.index_uid.as_str()).collect(); + let show_ranking_score = query.iter().any(|query| query.show_ranking_score); + let show_ranking_score_details = query.iter().any(|query| query.show_ranking_score_details); + Self { timestamp, total_received: 1, @@ -863,6 +870,8 @@ impl MultiSearchAggregator { total_distinct_index_count: distinct_indexes.len(), total_single_index: if distinct_indexes.len() == 1 { 1 } else { 0 }, total_search_count: query.len(), + show_ranking_score, + show_ranking_score_details, user_agents, } } @@ -884,6 +893,9 @@ impl MultiSearchAggregator { this.total_distinct_index_count.saturating_add(other.total_distinct_index_count); let total_single_index = this.total_single_index.saturating_add(other.total_single_index); let total_search_count = this.total_search_count.saturating_add(other.total_search_count); + let show_ranking_score = this.show_ranking_score || other.show_ranking_score; + let show_ranking_score_details = + this.show_ranking_score_details || other.show_ranking_score_details; let mut user_agents = this.user_agents; for user_agent in other.user_agents.into_iter() { @@ -899,6 +911,8 @@ impl MultiSearchAggregator { total_single_index, total_search_count, user_agents, + show_ranking_score, + show_ranking_score_details, // do not add _ or ..Default::default() here }; @@ -925,6 +939,10 @@ impl MultiSearchAggregator { "searches": { "total_search_count": self.total_search_count, "avg_search_count": (self.total_search_count as f64) / (self.total_received as f64), + }, + "scoring": { + "show_ranking_score": self.show_ranking_score, + "show_ranking_score_details": self.show_ranking_score_details, } }); From c30a14cb977c22e3fa8e02764fbde934e439668b Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Mon, 10 Jul 2023 13:12:12 +0200 Subject: [PATCH 27/54] Add telemetry --- meilisearch/src/analytics/segment_analytics.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 30796dc88..85f9e543a 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -574,6 +574,10 @@ pub struct SearchAggregator { filter_total_number_of_criteria: usize, used_syntax: HashMap, + // attributes_to_search_on + // every time a search is done using attributes_to_search_on + attributes_to_search_on_total_number_of_use: usize, + // q // The maximum number of terms in a q request max_terms_number: usize, @@ -647,6 +651,11 @@ impl SearchAggregator { ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count(); } + // attributes_to_search_on + if let Some(_) = query.attributes_to_search_on { + ret.attributes_to_search_on_total_number_of_use = 1; + } + if let Some(ref q) = query.q { ret.max_terms_number = q.split_whitespace().count(); } @@ -720,6 +729,11 @@ impl SearchAggregator { let used_syntax = self.used_syntax.entry(key).or_insert(0); *used_syntax = used_syntax.saturating_add(value); } + + // attributes_to_search_on + self.attributes_to_search_on_total_number_of_use += + other.attributes_to_search_on_total_number_of_use; + // q self.max_terms_number = self.max_terms_number.max(other.max_terms_number); @@ -786,6 +800,9 @@ impl SearchAggregator { "avg_criteria_number": format!("{:.2}", self.filter_sum_of_criteria_terms as f64 / self.filter_total_number_of_criteria as f64), "most_used_syntax": self.used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), }, + "attributes_to_search_on": { + "total_number_of_use": self.attributes_to_search_on_total_number_of_use, + }, "q": { "max_terms_number": self.max_terms_number, }, From d59e969c164bb17b0e191c3ed1f6c325d144ee15 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 10 Jul 2023 16:16:34 +0200 Subject: [PATCH 28/54] Allow a comma-separated value to the `vector` argument in GET search --- meilisearch/src/routes/indexes/search.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch/src/routes/indexes/search.rs b/meilisearch/src/routes/indexes/search.rs index 4d17adee6..3eabf61f3 100644 --- a/meilisearch/src/routes/indexes/search.rs +++ b/meilisearch/src/routes/indexes/search.rs @@ -35,7 +35,7 @@ pub struct SearchQueryGet { #[deserr(default, error = DeserrQueryParamError)] q: Option, #[deserr(default, error = DeserrQueryParamError)] - vector: Option>, + vector: Option>, #[deserr(default = Param(DEFAULT_SEARCH_OFFSET()), error = DeserrQueryParamError)] offset: Param, #[deserr(default = Param(DEFAULT_SEARCH_LIMIT()), error = DeserrQueryParamError)] @@ -88,7 +88,7 @@ impl From for SearchQuery { Self { q: other.q, - vector: other.vector, + vector: other.vector.map(CS::into_inner), offset: other.offset.0, limit: other.limit.0, page: other.page.as_deref().copied(), From 012c960fad5507d9c8456155203ced4251e1e73e Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 10 Jul 2023 16:50:37 +0200 Subject: [PATCH 29/54] Send the vector.max_vector_size telemetry --- meilisearch/src/analytics/segment_analytics.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 1a8843a37..35d02605d 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -723,6 +723,9 @@ impl SearchAggregator { // q self.max_terms_number = self.max_terms_number.max(other.max_terms_number); + // vector + self.max_vector_size = self.max_vector_size.max(other.max_vector_size); + // pagination self.max_limit = self.max_limit.max(other.max_limit); self.max_offset = self.max_offset.max(other.max_offset); @@ -789,6 +792,9 @@ impl SearchAggregator { "q": { "max_terms_number": self.max_terms_number, }, + "vector": { + "max_vector_size": self.max_vector_size, + }, "pagination": { "max_limit": self.max_limit, "max_offset": self.max_offset, From 74315b4ea83573e695585166f65d590511178d83 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 12 Jul 2023 10:08:29 +0200 Subject: [PATCH 30/54] Fixes #3911 --- milli/src/search/new/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/milli/src/search/new/mod.rs b/milli/src/search/new/mod.rs index 034b279ad..5da344014 100644 --- a/milli/src/search/new/mod.rs +++ b/milli/src/search/new/mod.rs @@ -85,7 +85,12 @@ impl<'ctx> SearchContext<'ctx> { let searchable_names = self.index.searchable_fields(self.txn)?; let mut restricted_fids = Vec::new(); + let mut contains_wildcard = false; for field_name in searchable_attributes { + if field_name == "*" { + contains_wildcard = true; + continue; + } let searchable_contains_name = searchable_names.as_ref().map(|sn| sn.iter().any(|name| name == field_name)); let fid = match (fids_map.id(field_name), searchable_contains_name) { @@ -132,7 +137,7 @@ impl<'ctx> SearchContext<'ctx> { restricted_fids.push(fid); } - self.restricted_fids = Some(restricted_fids); + self.restricted_fids = (!contains_wildcard).then_some(restricted_fids); Ok(()) } From 4310928803940181ae6cfd8a598d03910c5c525b Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 12 Jul 2023 10:08:56 +0200 Subject: [PATCH 31/54] Fixes #3912 --- milli/src/search/new/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/milli/src/search/new/mod.rs b/milli/src/search/new/mod.rs index 5da344014..26f992be2 100644 --- a/milli/src/search/new/mod.rs +++ b/milli/src/search/new/mod.rs @@ -104,8 +104,10 @@ impl<'ctx> SearchContext<'ctx> { } .into()) } + // The field is not searchable, but the searchableAttributes are set to * => ignore field + (None, None) => continue, // The field is not searchable => User error - _otherwise => { + (_fid, Some(false)) => { let mut valid_fields: BTreeSet<_> = fids_map.names().map(String::from).collect(); From 16c8437b28a8ba18f479300307b81235cb05d1d7 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 12 Jul 2023 10:36:55 +0200 Subject: [PATCH 32/54] Update tests --- meilisearch/tests/search/errors.rs | 51 +++++++++++++- .../tests/search/restrict_searchable.rs | 70 +++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/meilisearch/tests/search/errors.rs b/meilisearch/tests/search/errors.rs index 3d34de0fd..0745aa7d6 100644 --- a/meilisearch/tests/search/errors.rs +++ b/meilisearch/tests/search/errors.rs @@ -968,9 +968,12 @@ async fn sort_unset_ranking_rule() { async fn search_on_unknown_field() { let server = Server::new().await; let index = server.index("test"); + index.update_settings_searchable_attributes(json!(["id", "title"])).await; + index.wait_task(0).await; + let documents = DOCUMENTS.clone(); index.add_documents(documents, None).await; - index.wait_task(0).await; + index.wait_task(1).await; index .search( @@ -989,3 +992,49 @@ async fn search_on_unknown_field() { ) .await; } + +#[actix_rt::test] +async fn search_on_unknown_field_plus_joker() { + let server = Server::new().await; + let index = server.index("test"); + index.update_settings_searchable_attributes(json!(["id", "title"])).await; + index.wait_task(0).await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_task(1).await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["*", "unknown"]}), + |response, code| { + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r###" + { + "message": "Attribute `unknown` is not searchable. Available searchable attributes are: `id, title`.", + "code": "invalid_search_attributes_to_search_on", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_attributes_to_search_on" + } + "###); + }, + ) + .await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown", "*"]}), + |response, code| { + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r###" + { + "message": "Attribute `unknown` is not searchable. Available searchable attributes are: `id, title`.", + "code": "invalid_search_attributes_to_search_on", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_attributes_to_search_on" + } + "###); + }, + ) + .await; +} diff --git a/meilisearch/tests/search/restrict_searchable.rs b/meilisearch/tests/search/restrict_searchable.rs index f119acea5..6e41c5c55 100644 --- a/meilisearch/tests/search/restrict_searchable.rs +++ b/meilisearch/tests/search/restrict_searchable.rs @@ -49,6 +49,76 @@ async fn simple_search_on_title() { .await; } +#[actix_rt::test] +async fn search_no_searchable_attribute_set() { + let server = Server::new().await; + let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown"]}), + |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"0"); + }, + ) + .await; + + index.update_settings_searchable_attributes(json!(["*"])).await; + index.wait_task(1).await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown"]}), + |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"0"); + }, + ) + .await; + + index.update_settings_searchable_attributes(json!(["description", "*", "title"])).await; + index.wait_task(2).await; + + index + .search( + json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown"]}), + |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"0"); + }, + ) + .await; +} + +#[actix_rt::test] +async fn search_on_all_attributes() { + let server = Server::new().await; + let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await; + + index + .search(json!({"q": "Captain Marvel", "attributesToSearchOn": ["*"]}), |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"3"); + }) + .await; +} + +#[actix_rt::test] +async fn search_on_all_attributes_restricted_set() { + let server = Server::new().await; + let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await; + index.update_settings_searchable_attributes(json!(["title"])).await; + index.wait_task(1).await; + + index + .search(json!({"q": "Captain Marvel", "attributesToSearchOn": ["*"]}), |response, code| { + snapshot!(code, @"200 OK"); + snapshot!(response["hits"].as_array().unwrap().len(), @"2"); + }) + .await; +} + #[actix_rt::test] async fn simple_prefix_search_on_title() { let server = Server::new().await; From 183f23f40d4152a5387d6bb5e41e12de8bc563bd Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 12 Jul 2023 16:06:15 +0200 Subject: [PATCH 33/54] More relevant test Co-authored-by: Many the fish --- meilisearch/tests/search/restrict_searchable.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meilisearch/tests/search/restrict_searchable.rs b/meilisearch/tests/search/restrict_searchable.rs index 6e41c5c55..b589ccfb7 100644 --- a/meilisearch/tests/search/restrict_searchable.rs +++ b/meilisearch/tests/search/restrict_searchable.rs @@ -77,15 +77,15 @@ async fn search_no_searchable_attribute_set() { ) .await; - index.update_settings_searchable_attributes(json!(["description", "*", "title"])).await; + index.update_settings_searchable_attributes(json!(["*"])).await; index.wait_task(2).await; index .search( - json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown"]}), + json!({"q": "Captain Marvel", "attributesToSearchOn": ["unknown", "title"]}), |response, code| { snapshot!(code, @"200 OK"); - snapshot!(response["hits"].as_array().unwrap().len(), @"0"); + snapshot!(response["hits"].as_array().unwrap().len(), @"2"); }, ) .await; From 13e3f8faae4a36b5413567ed11e511bb414fde89 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Thu, 13 Jul 2023 11:34:50 +0200 Subject: [PATCH 34/54] Fix typo --- meilisearch/src/analytics/segment_analytics.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 85f9e543a..17ccc22be 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -576,7 +576,7 @@ pub struct SearchAggregator { // attributes_to_search_on // every time a search is done using attributes_to_search_on - attributes_to_search_on_total_number_of_use: usize, + attributes_to_search_on_total_number_of_uses: usize, // q // The maximum number of terms in a q request @@ -653,7 +653,7 @@ impl SearchAggregator { // attributes_to_search_on if let Some(_) = query.attributes_to_search_on { - ret.attributes_to_search_on_total_number_of_use = 1; + ret.attributes_to_search_on_total_number_of_uses = 1; } if let Some(ref q) = query.q { @@ -731,8 +731,8 @@ impl SearchAggregator { } // attributes_to_search_on - self.attributes_to_search_on_total_number_of_use += - other.attributes_to_search_on_total_number_of_use; + self.attributes_to_search_on_total_number_of_uses += + other.attributes_to_search_on_total_number_of_uses; // q self.max_terms_number = self.max_terms_number.max(other.max_terms_number); @@ -801,7 +801,7 @@ impl SearchAggregator { "most_used_syntax": self.used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), }, "attributes_to_search_on": { - "total_number_of_use": self.attributes_to_search_on_total_number_of_use, + "total_number_of_uses": self.attributes_to_search_on_total_number_of_uses, }, "q": { "max_terms_number": self.max_terms_number, From 359b90288d5b755cc6e14217aef18a6bf48daed8 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Thu, 13 Jul 2023 11:36:03 +0200 Subject: [PATCH 35/54] Use saturating add --- meilisearch/src/analytics/segment_analytics.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 17ccc22be..eca6f90d8 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -731,8 +731,9 @@ impl SearchAggregator { } // attributes_to_search_on - self.attributes_to_search_on_total_number_of_uses += - other.attributes_to_search_on_total_number_of_uses; + self.attributes_to_search_on_total_number_of_uses = self + .attributes_to_search_on_total_number_of_uses + .saturating_add(other.attributes_to_search_on_total_number_of_uses); // q self.max_terms_number = self.max_terms_number.max(other.max_terms_number); From 9c0691156f0b1c6ab0c4426eca159860d394be31 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Mon, 10 Jul 2023 14:01:14 +0200 Subject: [PATCH 36/54] Add tests --- meilisearch/tests/search/mod.rs | 179 ++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/meilisearch/tests/search/mod.rs b/meilisearch/tests/search/mod.rs index e6eae7cb1..1aa1d5805 100644 --- a/meilisearch/tests/search/mod.rs +++ b/meilisearch/tests/search/mod.rs @@ -876,3 +876,182 @@ async fn experimental_feature_vector_store() { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @"[]"); } + +#[cfg(feature = "default")] +#[actix_rt::test] +async fn camelcased_words() { + let server = Server::new().await; + let index = server.index("test"); + + // related to https://github.com/meilisearch/meilisearch/issues/3818 + let documents = json!([ + { "id": 0, "title": "DeLonghi" }, + { "id": 1, "title": "delonghi" }, + { "id": 2, "title": "TestAB" }, + { "id": 3, "title": "testab" }, + ]); + index.add_documents(documents, None).await; + index.wait_task(0).await; + + index + .search(json!({"q": "deLonghi"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 0, + "title": "DeLonghi" + }, + { + "id": 1, + "title": "delonghi" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "dellonghi"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 0, + "title": "DeLonghi" + }, + { + "id": 1, + "title": "delonghi" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "testa"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "testab"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "TestaB"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "Testab"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "TestAb"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "tetsab"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "testab" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "TetsAB"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "TestAB" + }, + { + "id": 3, + "title": "testab" + } + ] + "###); + }) + .await; +} From c106906f8f339dad14ad8fd72dd5213dcdd27653 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Wed, 12 Jul 2023 16:47:30 +0200 Subject: [PATCH 37/54] deactivate camelCase segmentation --- Cargo.lock | 7 --- meilisearch/tests/search/mod.rs | 52 ++++++++++++++++++- .../tests/search/restrict_searchable.rs | 2 +- milli/Cargo.toml | 2 +- 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb6105741..e8747dc7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -709,7 +709,6 @@ dependencies = [ "csv", "deunicode", "either", - "finl_unicode", "fst", "irg-kvariants", "jieba-rs", @@ -1443,12 +1442,6 @@ dependencies = [ "nom_locate", ] -[[package]] -name = "finl_unicode" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" - [[package]] name = "flate2" version = "1.0.26" diff --git a/meilisearch/tests/search/mod.rs b/meilisearch/tests/search/mod.rs index 1aa1d5805..9c80aed31 100644 --- a/meilisearch/tests/search/mod.rs +++ b/meilisearch/tests/search/mod.rs @@ -888,7 +888,8 @@ async fn camelcased_words() { { "id": 0, "title": "DeLonghi" }, { "id": 1, "title": "delonghi" }, { "id": 2, "title": "TestAB" }, - { "id": 3, "title": "testab" }, + { "id": 3, "title": "TestAb" }, + { "id": 4, "title": "testab" }, ]); index.add_documents(documents, None).await; index.wait_task(0).await; @@ -940,6 +941,10 @@ async fn camelcased_words() { }, { "id": 3, + "title": "TestAb" + }, + { + "id": 4, "title": "testab" } ] @@ -958,6 +963,10 @@ async fn camelcased_words() { }, { "id": 3, + "title": "TestAb" + }, + { + "id": 4, "title": "testab" } ] @@ -976,6 +985,10 @@ async fn camelcased_words() { }, { "id": 3, + "title": "TestAb" + }, + { + "id": 4, "title": "testab" } ] @@ -994,6 +1007,10 @@ async fn camelcased_words() { }, { "id": 3, + "title": "TestAb" + }, + { + "id": 4, "title": "testab" } ] @@ -1012,6 +1029,10 @@ async fn camelcased_words() { }, { "id": 3, + "title": "TestAb" + }, + { + "id": 4, "title": "testab" } ] @@ -1019,8 +1040,27 @@ async fn camelcased_words() { }) .await; + // with Typos index - .search(json!({"q": "tetsab"}), |response, code| { + .search(json!({"q": "dellonghi"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "id": 0, + "title": "DeLonghi" + }, + { + "id": 1, + "title": "delonghi" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "TetsAB"}), |response, code| { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" [ @@ -1030,6 +1070,10 @@ async fn camelcased_words() { }, { "id": 3, + "title": "TestAb" + }, + { + "id": 4, "title": "testab" } ] @@ -1048,6 +1092,10 @@ async fn camelcased_words() { }, { "id": 3, + "title": "TestAb" + }, + { + "id": 4, "title": "testab" } ] diff --git a/meilisearch/tests/search/restrict_searchable.rs b/meilisearch/tests/search/restrict_searchable.rs index f119acea5..309729fca 100644 --- a/meilisearch/tests/search/restrict_searchable.rs +++ b/meilisearch/tests/search/restrict_searchable.rs @@ -240,7 +240,7 @@ async fn exactness_ranking_rule_order() { }, { "title": "Captain Marvel", - "desc": "CaptainMarvel", + "desc": "Captain the Marvel", "id": "2", }]), ) diff --git a/milli/Cargo.toml b/milli/Cargo.toml index bc1d9b7ee..854d29141 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -81,7 +81,7 @@ md5 = "0.7.0" rand = { version = "0.8.5", features = ["small_rng"] } [features] -all-tokenizations = ["charabia/default"] +all-tokenizations = ["charabia/chinese", "charabia/hebrew", "charabia/japanese", "charabia/thai", "charabia/korean", "charabia/greek"] # Use POSIX semaphores instead of SysV semaphores in LMDB # For more information on this feature, see heed's Cargo.toml From f9d94c584513de76e7c656ab30b9beb6bc6bc3af Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 17 Jul 2023 18:28:03 +0200 Subject: [PATCH 38/54] Test geo sort with string lat/lng --- meilisearch/tests/search/geo.rs | 62 +++++++++++++++++++++++++++++++++ meilisearch/tests/search/mod.rs | 1 + 2 files changed, 63 insertions(+) create mode 100644 meilisearch/tests/search/geo.rs diff --git a/meilisearch/tests/search/geo.rs b/meilisearch/tests/search/geo.rs new file mode 100644 index 000000000..4253bc210 --- /dev/null +++ b/meilisearch/tests/search/geo.rs @@ -0,0 +1,62 @@ +use once_cell::sync::Lazy; +use serde_json::{json, Value}; + +use crate::common::Server; + +pub(self) static DOCUMENTS: Lazy = Lazy::new(|| { + json!([ + { + "id": 1, + "name": "Taco Truck", + "address": "444 Salsa Street, Burritoville", + "type": "Mexican", + "rating": 9, + "_geo": { + "lat": 34.0522, + "lng": -118.2437 + } + }, + { + "id": 2, + "name": "La Bella Italia", + "address": "456 Elm Street, Townsville", + "type": "Italian", + "rating": 9, + "_geo": { + "lat": "45.4777599", + "lng": "9.1967508" + } + }, + { + "id": 3, + "name": "Crêpe Truck", + "address": "2 Billig Avenue, Rouenville", + "type": "French", + "rating": 10 + } + ]) +}); + +#[actix_rt::test] +async fn geo_sort_with_geo_strings() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.update_settings_filterable_attributes(json!(["_geo"])).await; + index.update_settings_sortable_attributes(json!(["_geo"])).await; + index.add_documents(documents, None).await; + index.wait_task(2).await; + + index + .search( + json!({ + "filter": "_geoRadius(45.472735, 9.184019, 10000)", + "sort": ["_geoPoint(0.0, 0.0):asc"] + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + }, + ) + .await; +} diff --git a/meilisearch/tests/search/mod.rs b/meilisearch/tests/search/mod.rs index 9c80aed31..d560ba734 100644 --- a/meilisearch/tests/search/mod.rs +++ b/meilisearch/tests/search/mod.rs @@ -4,6 +4,7 @@ mod errors; mod facet_search; mod formatted; +mod geo; mod multi; mod pagination; mod restrict_searchable; From d383afc82b1489980202e95e54c460f0b92bff34 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 17 Jul 2023 18:24:24 +0200 Subject: [PATCH 39/54] Fix the geo sort when lat and lng are strings --- milli/src/search/new/distinct.rs | 2 +- milli/src/search/new/geo_sort.rs | 50 +++++++++++++++++++++----------- milli/src/search/new/mod.rs | 1 + 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/milli/src/search/new/distinct.rs b/milli/src/search/new/distinct.rs index fff96bd5d..e90ffe878 100644 --- a/milli/src/search/new/distinct.rs +++ b/milli/src/search/new/distinct.rs @@ -100,7 +100,7 @@ fn facet_number_values<'a>( } /// Return an iterator over each string value in the given field of the given document. -fn facet_string_values<'a>( +pub fn facet_string_values<'a>( docid: u32, field_id: u16, index: &Index, diff --git a/milli/src/search/new/geo_sort.rs b/milli/src/search/new/geo_sort.rs index dddb7f426..bd9546048 100644 --- a/milli/src/search/new/geo_sort.rs +++ b/milli/src/search/new/geo_sort.rs @@ -6,6 +6,7 @@ use heed::{RoPrefix, RoTxn}; use roaring::RoaringBitmap; use rstar::RTree; +use super::facet_string_values; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; use crate::heed_codec::facet::{FieldDocIdFacetCodec, OrderedF64Codec}; use crate::score_details::{self, ScoreDetails}; @@ -157,23 +158,7 @@ impl GeoSort { let mut documents = self .geo_candidates .iter() - .map(|id| -> Result<_> { - Ok(( - id, - [ - facet_number_values(id, lat, ctx.index, ctx.txn)? - .next() - .expect("A geo faceted document doesn't contain any lat")? - .0 - .2, - facet_number_values(id, lng, ctx.index, ctx.txn)? - .next() - .expect("A geo faceted document doesn't contain any lng")? - .0 - .2, - ], - )) - }) + .map(|id| -> Result<_> { Ok((id, geo_value(id, lat, lng, ctx.index, ctx.txn)?)) }) .collect::>>()?; // computing the distance between two points is expensive thus we cache the result documents @@ -185,6 +170,37 @@ impl GeoSort { } } +/// Extracts the lat and long values from a single document. +/// +/// If it is not able to find it in the facet number index it will extract it +/// from the facet string index and parse it as f64 (as the geo extraction behaves). +fn geo_value( + docid: u32, + field_lat: u16, + field_lng: u16, + index: &Index, + rtxn: &RoTxn, +) -> Result<[f64; 2]> { + let extract_geo = |geo_field: u16| -> Result { + match facet_number_values(docid, geo_field, index, rtxn)?.next() { + Some(Ok(((_, _, geo), ()))) => Ok(geo), + Some(Err(e)) => Err(e.into()), + None => match facet_string_values(docid, geo_field, index, rtxn)?.next() { + Some(Ok((_, geo))) => { + Ok(geo.parse::().expect("cannot parse geo field as f64")) + } + Some(Err(e)) => Err(e.into()), + None => panic!("A geo faceted document doesn't contain any lat or lng"), + }, + } + }; + + let lat = extract_geo(field_lat)?; + let lng = extract_geo(field_lng)?; + + Ok([lat, lng]) +} + impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for GeoSort { fn id(&self) -> String { "geo_sort".to_owned() diff --git a/milli/src/search/new/mod.rs b/milli/src/search/new/mod.rs index 26f992be2..c27e02514 100644 --- a/milli/src/search/new/mod.rs +++ b/milli/src/search/new/mod.rs @@ -42,6 +42,7 @@ use roaring::RoaringBitmap; use sort::Sort; use space::Neighbor; +use self::distinct::facet_string_values; use self::geo_sort::GeoSort; pub use self::geo_sort::Strategy as GeoSortStrategy; use self::graph_based_ranking_rule::Words; From 516d2df862519ddf5a261a6c5b62678d7125fd6d Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 18 Jul 2023 11:46:16 +0200 Subject: [PATCH 40/54] Stop computing the update files size --- meilisearch/src/routes/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index d14ddf48a..69bc44160 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -284,9 +284,6 @@ pub fn create_all_stats( used_database_size += index_scheduler.used_size()?; database_size += auth_controller.size()?; used_database_size += auth_controller.used_size()?; - let update_file_size = index_scheduler.compute_update_file_size()?; - database_size += update_file_size; - used_database_size += update_file_size; let stats = Stats { database_size, used_database_size, last_update: last_task, indexes }; Ok(stats) From 2afd10f96d0041a2a050e5674a7b3733f207edda Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Tue, 18 Jul 2023 14:49:36 +0200 Subject: [PATCH 41/54] Update mini-dashboard to version 0.2.8 --- meilisearch/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index 3fbc21d82..517de2129 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -141,5 +141,5 @@ thai = ["meilisearch-types/thai"] greek = ["meilisearch-types/greek"] [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.7/build.zip" -sha1 = "28b45bf772c84f9a6e16bc1689b393bfce8da7d6" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.8/build.zip" +sha1 = "e9eb6200f30d91769a19ca17c0904293ffca94a7" From 0497f934949b03df4bd0a43347bd98d549a029fb Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Wed, 19 Jul 2023 12:52:43 +0200 Subject: [PATCH 42/54] Update Charabia to the last version --- Cargo.lock | 154 +++++++++++++++++++++++++++-------------------- milli/Cargo.toml | 2 +- 2 files changed, 89 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8747dc7a..370841384 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -405,7 +405,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -416,7 +416,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -603,7 +603,7 @@ checksum = "fdde5c9cd29ebd706ce1b35600920a33550e402fc998a2e53ad3b42c3c47a192" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -700,9 +700,9 @@ dependencies = [ [[package]] name = "charabia" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb49850f555eb71aa6fc6d4d79420e81f4d89fa56e0e9c0f6d19aace2f56c554" +checksum = "57aa1b4a8dda126c03ebf2f7e31d16cfc8781c2fe80dedd1a33459efc3e07578" dependencies = [ "aho-corasick", "cow-utils", @@ -794,7 +794,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -1021,9 +1021,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" dependencies = [ "csv-core", "itoa", @@ -1342,7 +1342,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -1358,6 +1358,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -1537,7 +1543,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -1679,7 +1685,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1719,6 +1725,12 @@ dependencies = [ "ahash 0.7.6", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "heapless" version = "0.7.16" @@ -1952,6 +1964,16 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "inout" version = "0.1.3" @@ -2164,9 +2186,9 @@ dependencies = [ [[package]] name = "lindera-cc-cedict-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bf79b29a90bcd22036e494d6cc9ac3abe9ab604b21f3258ba6dc1ce501801" +checksum = "2d2e8f2ca97ddf952fe340642511b9c14b373cb2eef711d526bb8ef2ca0969b8" dependencies = [ "anyhow", "bincode", @@ -2183,9 +2205,9 @@ dependencies = [ [[package]] name = "lindera-compress" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f2e99e67736352bbb6ed1c273643975822505067ca32194b0981040bc50527a" +checksum = "f72b460559bcbe8a9cee85ea4a5056133ed3abf373031191589236e656d65b59" dependencies = [ "anyhow", "flate2", @@ -2194,9 +2216,9 @@ dependencies = [ [[package]] name = "lindera-core" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3935e966409156f22cb4b334b21b0dce84b7aa1cad62214b466489d249c8e5" +checksum = "f586eb8a9393c32d5525e0e9336a3727bd1329674740097126f3b0bff8a1a1ea" dependencies = [ "anyhow", "bincode", @@ -2211,9 +2233,9 @@ dependencies = [ [[package]] name = "lindera-decompress" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7476406abb63c49d7f59c88b9b868ee8d2981495ea7e2c3ad129902f9916b3c6" +checksum = "1fb1facd8da698072fcc7338bd757730db53d59f313f44dd583fa03681dcc0e1" dependencies = [ "anyhow", "flate2", @@ -2222,9 +2244,9 @@ dependencies = [ [[package]] name = "lindera-dictionary" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808b7d2b3cabc25a4022526d484a4cfd1d5924dc76a26e0379707698841acef2" +checksum = "ec7be7410b1da7017a8948986b87af67082f605e9a716f0989790d795d677f0c" dependencies = [ "anyhow", "bincode", @@ -2242,9 +2264,9 @@ dependencies = [ [[package]] name = "lindera-ipadic-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f373a280958c930e5ee4a1e4db3a0ee0542afaf02d3b5cacb8cab4e298648e" +checksum = "705d07f8a45d04fd95149f7ad41a26d1f9e56c9c00402be6f9dd05e3d88b99c6" dependencies = [ "anyhow", "bincode", @@ -2263,9 +2285,9 @@ dependencies = [ [[package]] name = "lindera-ipadic-neologd-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92eff98e9ed1a7a412b91709c2343457a04ef02fa0c27c27e3a5892f5591eae9" +checksum = "633a93983ba13fba42328311a501091bd4a7aff0c94ae9eaa9d4733dd2b0468a" dependencies = [ "anyhow", "bincode", @@ -2284,9 +2306,9 @@ dependencies = [ [[package]] name = "lindera-ko-dic" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74c6d5bf7d8092bd6d10de7a5d74b70ea7cf234586235b0d6cdb903b05a6c9e2" +checksum = "a428e0d316b6c86f51bd919479692bc41ad840dba266ebc044663970f431ea18" dependencies = [ "bincode", "byteorder", @@ -2301,9 +2323,9 @@ dependencies = [ [[package]] name = "lindera-ko-dic-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a4add6d3c1e41ec9e2690d33e287d0223fb59a30ccee4980c23f31368cae1e" +checksum = "2a5288704c6b8a069c0a1705c38758e836497698b50453373ab3d56c6f9a7ef8" dependencies = [ "anyhow", "bincode", @@ -2321,9 +2343,9 @@ dependencies = [ [[package]] name = "lindera-tokenizer" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6a8acbd068019d1cdac7316f0dcb87f8e33ede2b13aa237f45114f9750afb8" +checksum = "106ba439b2e87529d9bbedbb88d69f635baba1195c26502b308f55a85885fc81" dependencies = [ "bincode", "byteorder", @@ -2336,9 +2358,9 @@ dependencies = [ [[package]] name = "lindera-unidic" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14abf0613d350b30d3b0406a33b1de8fa8d829f26516909421702174785991c8" +checksum = "3399b6dcfe1701333451d184ff3c677f433b320153427b146360c9e4bd8cb816" dependencies = [ "bincode", "byteorder", @@ -2353,9 +2375,9 @@ dependencies = [ [[package]] name = "lindera-unidic-builder" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e204ed53d9bd63227d1e6a6c1f122ca039e00a8634ac32e7fb0281eeec8615c4" +checksum = "b698227fdaeac32289173ab389b990d4eb00a40cbc9912020f69a0c491dabf55" dependencies = [ "anyhow", "bincode", @@ -2435,9 +2457,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "logging_timer" @@ -2470,7 +2492,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -2527,7 +2549,7 @@ dependencies = [ "hex", "http", "index-scheduler", - "indexmap", + "indexmap 1.9.3", "insta", "is-terminal", "itertools", @@ -2680,7 +2702,7 @@ dependencies = [ "grenad", "heed", "hnsw", - "indexmap", + "indexmap 1.9.3", "insta", "itertools", "json-depth-checker", @@ -2866,9 +2888,9 @@ checksum = "f69e48cd7c8e5bb52a1da1287fdbfd877c32673176583ce664cd63b201aba385" [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oorandom" @@ -3025,7 +3047,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -3170,9 +3192,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.59" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -3215,9 +3237,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "quote" -version = "1.0.28" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" dependencies = [ "proc-macro2", ] @@ -3578,9 +3600,9 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.163" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] @@ -3596,22 +3618,22 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" dependencies = [ - "indexmap", + "indexmap 2.0.0", "itoa", "ryu", "serde", @@ -3832,9 +3854,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" dependencies = [ "proc-macro2", "quote", @@ -3921,22 +3943,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -4018,7 +4040,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", ] [[package]] @@ -4094,7 +4116,7 @@ version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" dependencies = [ - "indexmap", + "indexmap 1.9.3", "serde", "serde_spanned", "toml_datetime", @@ -4343,7 +4365,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", "wasm-bindgen-shared", ] @@ -4377,7 +4399,7 @@ checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.26", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/milli/Cargo.toml b/milli/Cargo.toml index 854d29141..2689975cd 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -17,7 +17,7 @@ bincode = "1.3.3" bstr = "1.4.0" bytemuck = { version = "1.13.1", features = ["extern_crate_alloc"] } byteorder = "1.4.3" -charabia = { version = "0.8.1", default-features = false } +charabia = { version = "0.8.2", default-features = false } concat-arrays = "0.1.2" crossbeam-channel = "0.5.8" deserr = "0.5.0" From 54ae1b5a671fe5ac64b582cf4a769ad2ec2ce0d5 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 20 Jul 2023 14:11:17 +0200 Subject: [PATCH 43/54] Update mini-dashboard to version 0.2.9 --- meilisearch/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index 517de2129..b048c9720 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -141,5 +141,5 @@ thai = ["meilisearch-types/thai"] greek = ["meilisearch-types/greek"] [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.8/build.zip" -sha1 = "e9eb6200f30d91769a19ca17c0904293ffca94a7" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.9/build.zip" +sha1 = "9c713298788363e41087f2b40e7509d390033e30" From df528b41d8deebb6fd98c4e55e774cb49004ea57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 20 Jul 2023 17:57:07 +0200 Subject: [PATCH 44/54] Normalize for the search the facets values --- milli/src/heed_codec/beu16_str_codec.rs | 27 ++++++ milli/src/heed_codec/mod.rs | 2 + milli/src/index.rs | 14 ++- milli/src/lib.rs | 7 +- milli/src/search/mod.rs | 49 ++++++++--- milli/src/update/clear_documents.rs | 2 + milli/src/update/delete_documents.rs | 1 + milli/src/update/facet/mod.rs | 87 ++++++++++++++----- .../helpers/merge_functions.rs | 22 +++++ .../src/update/index_documents/helpers/mod.rs | 6 +- milli/src/update/index_documents/mod.rs | 2 +- milli/src/update/mod.rs | 5 +- 12 files changed, 178 insertions(+), 46 deletions(-) create mode 100644 milli/src/heed_codec/beu16_str_codec.rs diff --git a/milli/src/heed_codec/beu16_str_codec.rs b/milli/src/heed_codec/beu16_str_codec.rs new file mode 100644 index 000000000..d1b85d47f --- /dev/null +++ b/milli/src/heed_codec/beu16_str_codec.rs @@ -0,0 +1,27 @@ +use std::borrow::Cow; +use std::convert::TryInto; +use std::str; + +pub struct BEU16StrCodec; + +impl<'a> heed::BytesDecode<'a> for BEU16StrCodec { + type DItem = (u16, &'a str); + + fn bytes_decode(bytes: &'a [u8]) -> Option { + let (n_bytes, str_bytes) = bytes.split_at(2); + let n = n_bytes.try_into().map(u16::from_be_bytes).ok()?; + let s = str::from_utf8(str_bytes).ok()?; + Some((n, s)) + } +} + +impl<'a> heed::BytesEncode<'a> for BEU16StrCodec { + type EItem = (u16, &'a str); + + fn bytes_encode((n, s): &Self::EItem) -> Option> { + let mut bytes = Vec::with_capacity(s.len() + 2); + bytes.extend_from_slice(&n.to_be_bytes()); + bytes.extend_from_slice(s.as_bytes()); + Some(Cow::Owned(bytes)) + } +} diff --git a/milli/src/heed_codec/mod.rs b/milli/src/heed_codec/mod.rs index 666f68e28..d04eaa644 100644 --- a/milli/src/heed_codec/mod.rs +++ b/milli/src/heed_codec/mod.rs @@ -1,3 +1,4 @@ +mod beu16_str_codec; mod beu32_str_codec; mod byte_slice_ref; pub mod facet; @@ -14,6 +15,7 @@ mod str_str_u8_codec; pub use byte_slice_ref::ByteSliceRefCodec; pub use str_ref::StrRefCodec; +pub use self::beu16_str_codec::BEU16StrCodec; pub use self::beu32_str_codec::BEU32StrCodec; pub use self::field_id_word_count_codec::FieldIdWordCountCodec; pub use self::fst_set_codec::FstSetCodec; diff --git a/milli/src/index.rs b/milli/src/index.rs index 392ed1705..544047293 100644 --- a/milli/src/index.rs +++ b/milli/src/index.rs @@ -1,5 +1,5 @@ use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fs::File; use std::mem::size_of; use std::path::Path; @@ -21,7 +21,9 @@ use crate::heed_codec::facet::{ FacetGroupKeyCodec, FacetGroupValueCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetStringCodec, FieldIdCodec, OrderedF64Codec, }; -use crate::heed_codec::{FstSetCodec, ScriptLanguageCodec, StrBEU16Codec, StrRefCodec}; +use crate::heed_codec::{ + BEU16StrCodec, FstSetCodec, ScriptLanguageCodec, StrBEU16Codec, StrRefCodec, +}; use crate::readable_slices::ReadableSlices; use crate::{ default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds, @@ -96,6 +98,7 @@ pub mod db_name { pub const FACET_ID_IS_NULL_DOCIDS: &str = "facet-id-is-null-docids"; pub const FACET_ID_IS_EMPTY_DOCIDS: &str = "facet-id-is-empty-docids"; pub const FACET_ID_STRING_DOCIDS: &str = "facet-id-string-docids"; + pub const FACET_ID_NORMALIZED_STRING_STRINGS: &str = "facet-id-normalized-string-strings"; pub const FACET_ID_STRING_FST: &str = "facet-id-string-fst"; pub const FIELD_ID_DOCID_FACET_F64S: &str = "field-id-docid-facet-f64s"; pub const FIELD_ID_DOCID_FACET_STRINGS: &str = "field-id-docid-facet-strings"; @@ -157,6 +160,8 @@ pub struct Index { pub facet_id_f64_docids: Database, FacetGroupValueCodec>, /// Maps the facet field id and ranges of strings with the docids that corresponds to them. pub facet_id_string_docids: Database, FacetGroupValueCodec>, + /// Maps the facet field id of the normalized-for-search string facets with their original versions. + pub facet_id_normalized_string_strings: Database>>, /// Maps the facet field id of the string facets with an FST containing all the facets values. pub facet_id_string_fst: Database, FstSetCodec>, @@ -181,7 +186,7 @@ impl Index { ) -> Result { use db_name::*; - options.max_dbs(24); + options.max_dbs(25); unsafe { options.flag(Flags::MdbAlwaysFreePages) }; let env = options.open(path)?; @@ -211,6 +216,8 @@ impl Index { let facet_id_f64_docids = env.create_database(&mut wtxn, Some(FACET_ID_F64_DOCIDS))?; let facet_id_string_docids = env.create_database(&mut wtxn, Some(FACET_ID_STRING_DOCIDS))?; + let facet_id_normalized_string_strings = + env.create_database(&mut wtxn, Some(FACET_ID_NORMALIZED_STRING_STRINGS))?; let facet_id_string_fst = env.create_database(&mut wtxn, Some(FACET_ID_STRING_FST))?; let facet_id_exists_docids = env.create_database(&mut wtxn, Some(FACET_ID_EXISTS_DOCIDS))?; @@ -246,6 +253,7 @@ impl Index { field_id_word_count_docids, facet_id_f64_docids, facet_id_string_docids, + facet_id_normalized_string_strings, facet_id_string_fst, facet_id_exists_docids, facet_id_is_null_docids, diff --git a/milli/src/lib.rs b/milli/src/lib.rs index 55b283931..3e5f63fd5 100644 --- a/milli/src/lib.rs +++ b/milli/src/lib.rs @@ -51,9 +51,10 @@ pub use self::error::{ pub use self::external_documents_ids::ExternalDocumentsIds; pub use self::fields_ids_map::FieldsIdsMap; pub use self::heed_codec::{ - BEU32StrCodec, BoRoaringBitmapCodec, BoRoaringBitmapLenCodec, CboRoaringBitmapCodec, - CboRoaringBitmapLenCodec, FieldIdWordCountCodec, ObkvCodec, RoaringBitmapCodec, - RoaringBitmapLenCodec, StrBEU32Codec, U8StrStrCodec, UncheckedU8StrStrCodec, + BEU16StrCodec, BEU32StrCodec, BoRoaringBitmapCodec, BoRoaringBitmapLenCodec, + CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, FieldIdWordCountCodec, ObkvCodec, + RoaringBitmapCodec, RoaringBitmapLenCodec, StrBEU32Codec, U8StrStrCodec, + UncheckedU8StrStrCodec, }; pub use self::index::Index; pub use self::search::{ diff --git a/milli/src/search/mod.rs b/milli/src/search/mod.rs index 65e78caa9..fe34b38e1 100644 --- a/milli/src/search/mod.rs +++ b/milli/src/search/mod.rs @@ -339,11 +339,12 @@ impl<'a> SearchForFacetValues<'a> { let mut stream = fst.search(automaton).into_stream(); let mut length = 0; - while let Some(facet_value) = stream.next() { + 'outer: while let Some(facet_value) = stream.next() { let value = std::str::from_utf8(facet_value)?; - let key = FacetGroupKey { field_id: fid, level: 0, left_bound: value }; - let docids = match index.facet_id_string_docids.get(rtxn, &key)? { - Some(FacetGroupValue { bitmap, .. }) => bitmap, + let database = index.facet_id_normalized_string_strings; + let key = (fid, value); + let original_strings = match database.get(rtxn, &key)? { + Some(original_strings) => original_strings, None => { error!( "the facet value is missing from the facet database: {key:?}" @@ -351,16 +352,36 @@ impl<'a> SearchForFacetValues<'a> { continue; } }; - let count = search_candidates.intersection_len(&docids); - if count != 0 { - let value = self - .one_original_value_of(fid, value, docids.min().unwrap())? - .unwrap_or_else(|| query.to_string()); - results.push(FacetValueHit { value, count }); - length += 1; - } - if length >= MAX_NUMBER_OF_FACETS { - break; + for original_string in original_strings { + let key = FacetGroupKey { + field_id: fid, + level: 0, + left_bound: original_string.as_str(), + }; + let docids = match index.facet_id_string_docids.get(rtxn, &key)? { + Some(FacetGroupValue { bitmap, .. }) => bitmap, + None => { + error!( + "the facet value is missing from the facet database: {key:?}" + ); + continue; + } + }; + let count = search_candidates.intersection_len(&docids); + if count != 0 { + let value = self + .one_original_value_of( + fid, + &original_string, + docids.min().unwrap(), + )? + .unwrap_or_else(|| query.to_string()); + results.push(FacetValueHit { value, count }); + length += 1; + } + if length >= MAX_NUMBER_OF_FACETS { + break 'outer; + } } } } diff --git a/milli/src/update/clear_documents.rs b/milli/src/update/clear_documents.rs index 5fdf8ef49..2d2f78d5e 100644 --- a/milli/src/update/clear_documents.rs +++ b/milli/src/update/clear_documents.rs @@ -34,6 +34,7 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> { script_language_docids, facet_id_f64_docids, facet_id_string_docids, + facet_id_normalized_string_strings, facet_id_string_fst, facet_id_exists_docids, facet_id_is_null_docids, @@ -92,6 +93,7 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> { word_prefix_fid_docids.clear(self.wtxn)?; script_language_docids.clear(self.wtxn)?; facet_id_f64_docids.clear(self.wtxn)?; + facet_id_normalized_string_strings.clear(self.wtxn)?; facet_id_string_fst.clear(self.wtxn)?; facet_id_exists_docids.clear(self.wtxn)?; facet_id_is_null_docids.clear(self.wtxn)?; diff --git a/milli/src/update/delete_documents.rs b/milli/src/update/delete_documents.rs index c9124e591..a412291ab 100644 --- a/milli/src/update/delete_documents.rs +++ b/milli/src/update/delete_documents.rs @@ -237,6 +237,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { word_prefix_fid_docids, facet_id_f64_docids: _, facet_id_string_docids: _, + facet_id_normalized_string_strings: _, facet_id_string_fst: _, field_id_docid_facet_f64s: _, field_id_docid_facet_strings: _, diff --git a/milli/src/update/facet/mod.rs b/milli/src/update/facet/mod.rs index 0e6fd494c..16fc1cd2f 100644 --- a/milli/src/update/facet/mod.rs +++ b/milli/src/update/facet/mod.rs @@ -76,9 +76,14 @@ pub const FACET_MAX_GROUP_SIZE: u8 = 8; pub const FACET_GROUP_SIZE: u8 = 4; pub const FACET_MIN_LEVEL_SIZE: u8 = 5; +use std::collections::BTreeSet; use std::fs::File; +use std::iter::FromIterator; -use heed::types::DecodeIgnore; +use charabia::normalizer::{Normalize, NormalizerOption}; +use grenad::{CompressionType, SortAlgorithm}; +use heed::types::{ByteSlice, DecodeIgnore, SerdeJson}; +use heed::BytesEncode; use log::debug; use time::OffsetDateTime; @@ -87,7 +92,9 @@ use super::FacetsUpdateBulk; use crate::facet::FacetType; use crate::heed_codec::facet::{FacetGroupKey, FacetGroupKeyCodec, FacetGroupValueCodec}; use crate::heed_codec::ByteSliceRefCodec; -use crate::{Index, Result, BEU16}; +use crate::update::index_documents::create_sorter; +use crate::update::merge_btreeset_string; +use crate::{BEU16StrCodec, Index, Result, BEU16}; pub mod bulk; pub mod delete; @@ -159,26 +166,69 @@ impl<'i> FacetsUpdate<'i> { incremental_update.execute(wtxn)?; } + // We clear the list of normalized-for-search facets + // and the previous FSTs to compute everything from scratch + self.index.facet_id_normalized_string_strings.clear(wtxn)?; + self.index.facet_id_string_fst.clear(wtxn)?; + + // As we can't use the same write transaction to read and write in two different databases + // we must create a temporary sorter that we will write into LMDB afterward. + // As multiple unnormalized facet values can become the same normalized facet value + // we must merge them together. + let mut sorter = create_sorter( + SortAlgorithm::Unstable, + merge_btreeset_string, + CompressionType::None, + None, + None, + None, + ); + + // We iterate on the list of original, semi-normalized, facet values + // and normalize them for search, inserting them in LMDB in any given order. + let options = NormalizerOption { lossy: true, ..Default::default() }; + let database = self.index.facet_id_string_docids.remap_data_type::(); + for result in database.iter(wtxn)? { + let (facet_group_key, ()) = result?; + if let FacetGroupKey { field_id, level: 0, left_bound } = facet_group_key { + let normalized_facet = left_bound.normalize(&options); + let set = BTreeSet::from_iter(std::iter::once(left_bound)); + let key = (field_id, normalized_facet.as_ref()); + let key = BEU16StrCodec::bytes_encode(&key).ok_or(heed::Error::Encoding)?; + let val = SerdeJson::bytes_encode(&set).ok_or(heed::Error::Encoding)?; + sorter.insert(key, val)?; + } + } + + // In this loop we don't need to take care of merging bitmaps + // as the grenad sorter already merged them for us. + let mut merger_iter = sorter.into_stream_merger_iter()?; + while let Some((key_bytes, btreeset_bytes)) = merger_iter.next()? { + self.index + .facet_id_normalized_string_strings + .remap_types::() + .put(wtxn, key_bytes, btreeset_bytes)?; + } + // We compute one FST by string facet let mut text_fsts = vec![]; let mut current_fst: Option<(u16, fst::SetBuilder>)> = None; - let database = self.index.facet_id_string_docids.remap_data_type::(); + let database = + self.index.facet_id_normalized_string_strings.remap_data_type::(); for result in database.iter(wtxn)? { - let (facet_group_key, _) = result?; - if let FacetGroupKey { field_id, level: 0, left_bound } = facet_group_key { - current_fst = match current_fst.take() { - Some((fid, fst_builder)) if fid != field_id => { - let fst = fst_builder.into_set(); - text_fsts.push((fid, fst)); - Some((field_id, fst::SetBuilder::memory())) - } - Some((field_id, fst_builder)) => Some((field_id, fst_builder)), - None => Some((field_id, fst::SetBuilder::memory())), - }; - - if let Some((_, fst_builder)) = current_fst.as_mut() { - fst_builder.insert(left_bound)?; + let ((field_id, normalized_facet), _) = result?; + current_fst = match current_fst.take() { + Some((fid, fst_builder)) if fid != field_id => { + let fst = fst_builder.into_set(); + text_fsts.push((fid, fst)); + Some((field_id, fst::SetBuilder::memory())) } + Some((field_id, fst_builder)) => Some((field_id, fst_builder)), + None => Some((field_id, fst::SetBuilder::memory())), + }; + + if let Some((_, fst_builder)) = current_fst.as_mut() { + fst_builder.insert(normalized_facet)?; } } @@ -187,9 +237,6 @@ impl<'i> FacetsUpdate<'i> { text_fsts.push((field_id, fst)); } - // We remove all of the previous FSTs that were in this database - self.index.facet_id_string_fst.clear(wtxn)?; - // We write those FSTs in LMDB now for (field_id, fst) in text_fsts { self.index.facet_id_string_fst.put(wtxn, &BEU16::new(field_id), &fst)?; diff --git a/milli/src/update/index_documents/helpers/merge_functions.rs b/milli/src/update/index_documents/helpers/merge_functions.rs index 64bee95df..5d111067a 100644 --- a/milli/src/update/index_documents/helpers/merge_functions.rs +++ b/milli/src/update/index_documents/helpers/merge_functions.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::collections::BTreeSet; use std::io; use std::result::Result as StdResult; @@ -44,6 +45,27 @@ pub fn merge_roaring_bitmaps<'a>(_key: &[u8], values: &[Cow<'a, [u8]>]) -> Resul } } +pub fn merge_btreeset_string<'a>(_key: &[u8], values: &[Cow<'a, [u8]>]) -> Result> { + if values.len() == 1 { + Ok(values[0].clone()) + } else { + // TODO improve the perf by using a `#[borrow] Cow`. + let strings: BTreeSet = values + .iter() + .map(AsRef::as_ref) + .map(serde_json::from_slice::>) + .map(StdResult::unwrap) + .reduce(|mut current, new| { + for x in new { + current.insert(x); + } + current + }) + .unwrap(); + Ok(Cow::Owned(serde_json::to_vec(&strings).unwrap())) + } +} + pub fn keep_first<'a>(_key: &[u8], values: &[Cow<'a, [u8]>]) -> Result> { Ok(values[0].clone()) } diff --git a/milli/src/update/index_documents/helpers/mod.rs b/milli/src/update/index_documents/helpers/mod.rs index 95e497af4..d59a3bc08 100644 --- a/milli/src/update/index_documents/helpers/mod.rs +++ b/milli/src/update/index_documents/helpers/mod.rs @@ -13,9 +13,9 @@ pub use grenad_helpers::{ GrenadParameters, MergeableReader, }; pub use merge_functions::{ - concat_u32s_array, keep_first, keep_latest_obkv, merge_cbo_roaring_bitmaps, - merge_obkvs_and_operations, merge_roaring_bitmaps, merge_two_obkvs, serialize_roaring_bitmap, - MergeFn, + concat_u32s_array, keep_first, keep_latest_obkv, merge_btreeset_string, + merge_cbo_roaring_bitmaps, merge_obkvs_and_operations, merge_roaring_bitmaps, merge_two_obkvs, + serialize_roaring_bitmap, MergeFn, }; use crate::MAX_WORD_LENGTH; diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 20a36237e..849e84035 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -26,7 +26,7 @@ pub use self::enrich::{ }; pub use self::helpers::{ as_cloneable_grenad, create_sorter, create_writer, fst_stream_into_hashset, - fst_stream_into_vec, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, + fst_stream_into_vec, merge_btreeset_string, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, sorter_into_lmdb_database, valid_lmdb_key, writer_into_reader, ClonableMmap, MergeFn, }; use self::helpers::{grenad_obkv_into_chunks, GrenadParameters}; diff --git a/milli/src/update/mod.rs b/milli/src/update/mod.rs index 32584825b..9982957e5 100644 --- a/milli/src/update/mod.rs +++ b/milli/src/update/mod.rs @@ -4,8 +4,9 @@ pub use self::delete_documents::{DeleteDocuments, DeletionStrategy, DocumentDele pub use self::facet::bulk::FacetsUpdateBulk; pub use self::facet::incremental::FacetsUpdateIncrementalInner; pub use self::index_documents::{ - merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, DocumentAdditionResult, DocumentId, - IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, MergeFn, + merge_btreeset_string, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, + DocumentAdditionResult, DocumentId, IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, + MergeFn, }; pub use self::indexer_config::IndexerConfig; pub use self::prefix_word_pairs::{ From 691a5368937a43e00248129641c989dd59ee2fff Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 24 Jul 2023 17:52:08 +0200 Subject: [PATCH 45/54] Implement the facet search with the normalized index --- milli/src/search/mod.rs | 174 ++++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 85 deletions(-) diff --git a/milli/src/search/mod.rs b/milli/src/search/mod.rs index fe34b38e1..e1ee37d79 100644 --- a/milli/src/search/mod.rs +++ b/milli/src/search/mod.rs @@ -1,5 +1,8 @@ use std::fmt; +use std::ops::ControlFlow; +use charabia::normalizer::NormalizerOption; +use charabia::Normalize; use fst::automaton::{Automaton, Str}; use fst::{IntoStreamer, Streamer}; use levenshtein_automata::{LevenshteinAutomatonBuilder as LevBuilder, DFA}; @@ -14,8 +17,8 @@ use crate::error::UserError; use crate::heed_codec::facet::{FacetGroupKey, FacetGroupValue}; use crate::score_details::{ScoreDetails, ScoringStrategy}; use crate::{ - execute_search, normalize_facet, AscDesc, DefaultSearchLogger, DocumentId, FieldId, Index, - Result, SearchContext, BEU16, + execute_search, AscDesc, DefaultSearchLogger, DocumentId, FieldId, Index, Result, + SearchContext, BEU16, }; // Building these factories is not free. @@ -301,29 +304,28 @@ impl<'a> SearchForFacetValues<'a> { match self.query.as_ref() { Some(query) => { - let query = normalize_facet(query); - let query = query.as_str(); + let options = NormalizerOption { lossy: true, ..Default::default() }; + let query = query.normalize(&options); + let query = query.as_ref(); + let authorize_typos = self.search_query.index.authorize_typos(rtxn)?; let field_authorizes_typos = !self.search_query.index.exact_attributes_ids(rtxn)?.contains(&fid); if authorize_typos && field_authorizes_typos { - let mut results = vec![]; - let exact_words_fst = self.search_query.index.exact_words(rtxn)?; if exact_words_fst.map_or(false, |fst| fst.contains(query)) { - let key = FacetGroupKey { field_id: fid, level: 0, left_bound: query }; - if let Some(FacetGroupValue { bitmap, .. }) = - index.facet_id_string_docids.get(rtxn, &key)? - { - let count = search_candidates.intersection_len(&bitmap); - if count != 0 { - let value = self - .one_original_value_of(fid, query, bitmap.min().unwrap())? - .unwrap_or_else(|| query.to_string()); - results.push(FacetValueHit { value, count }); - } + let mut results = vec![]; + if fst.contains(query) { + self.fetch_original_facets_using_normalized( + fid, + query, + query, + &search_candidates, + &mut results, + )?; } + Ok(results) } else { let one_typo = self.search_query.index.min_word_len_one_typo(rtxn)?; let two_typos = self.search_query.index.min_word_len_two_typos(rtxn)?; @@ -338,81 +340,41 @@ impl<'a> SearchForFacetValues<'a> { }; let mut stream = fst.search(automaton).into_stream(); - let mut length = 0; - 'outer: while let Some(facet_value) = stream.next() { + let mut results = vec![]; + while let Some(facet_value) = stream.next() { let value = std::str::from_utf8(facet_value)?; - let database = index.facet_id_normalized_string_strings; - let key = (fid, value); - let original_strings = match database.get(rtxn, &key)? { - Some(original_strings) => original_strings, - None => { - error!( - "the facet value is missing from the facet database: {key:?}" - ); - continue; - } - }; - for original_string in original_strings { - let key = FacetGroupKey { - field_id: fid, - level: 0, - left_bound: original_string.as_str(), - }; - let docids = match index.facet_id_string_docids.get(rtxn, &key)? { - Some(FacetGroupValue { bitmap, .. }) => bitmap, - None => { - error!( - "the facet value is missing from the facet database: {key:?}" - ); - continue; - } - }; - let count = search_candidates.intersection_len(&docids); - if count != 0 { - let value = self - .one_original_value_of( - fid, - &original_string, - docids.min().unwrap(), - )? - .unwrap_or_else(|| query.to_string()); - results.push(FacetValueHit { value, count }); - length += 1; - } - if length >= MAX_NUMBER_OF_FACETS { - break 'outer; - } + if self + .fetch_original_facets_using_normalized( + fid, + value, + query, + &search_candidates, + &mut results, + )? + .is_break() + { + break; } } - } - Ok(results) + Ok(results) + } } else { let automaton = Str::new(query).starts_with(); let mut stream = fst.search(automaton).into_stream(); let mut results = vec![]; - let mut length = 0; while let Some(facet_value) = stream.next() { let value = std::str::from_utf8(facet_value)?; - let key = FacetGroupKey { field_id: fid, level: 0, left_bound: value }; - let docids = match index.facet_id_string_docids.get(rtxn, &key)? { - Some(FacetGroupValue { bitmap, .. }) => bitmap, - None => { - error!( - "the facet value is missing from the facet database: {key:?}" - ); - continue; - } - }; - let count = search_candidates.intersection_len(&docids); - if count != 0 { - let value = self - .one_original_value_of(fid, value, docids.min().unwrap())? - .unwrap_or_else(|| query.to_string()); - results.push(FacetValueHit { value, count }); - length += 1; - } - if length >= MAX_NUMBER_OF_FACETS { + if self + .fetch_original_facets_using_normalized( + fid, + value, + query, + &search_candidates, + &mut results, + )? + .is_break() + { break; } } @@ -422,7 +384,6 @@ impl<'a> SearchForFacetValues<'a> { } None => { let mut results = vec![]; - let mut length = 0; let prefix = FacetGroupKey { field_id: fid, level: 0, left_bound: "" }; for result in index.facet_id_string_docids.prefix_iter(rtxn, &prefix)? { let (FacetGroupKey { left_bound, .. }, FacetGroupValue { bitmap, .. }) = @@ -433,9 +394,8 @@ impl<'a> SearchForFacetValues<'a> { .one_original_value_of(fid, left_bound, bitmap.min().unwrap())? .unwrap_or_else(|| left_bound.to_string()); results.push(FacetValueHit { value, count }); - length += 1; } - if length >= MAX_NUMBER_OF_FACETS { + if results.len() >= MAX_NUMBER_OF_FACETS { break; } } @@ -443,6 +403,50 @@ impl<'a> SearchForFacetValues<'a> { } } } + + fn fetch_original_facets_using_normalized( + &self, + fid: FieldId, + value: &str, + query: &str, + search_candidates: &RoaringBitmap, + results: &mut Vec, + ) -> Result> { + let index = self.search_query.index; + let rtxn = self.search_query.rtxn; + + let database = index.facet_id_normalized_string_strings; + let key = (fid, value); + let original_strings = match database.get(rtxn, &key)? { + Some(original_strings) => original_strings, + None => { + error!("the facet value is missing from the facet database: {key:?}"); + return Ok(ControlFlow::Continue(())); + } + }; + for original in original_strings { + let key = FacetGroupKey { field_id: fid, level: 0, left_bound: original.as_str() }; + let docids = match index.facet_id_string_docids.get(rtxn, &key)? { + Some(FacetGroupValue { bitmap, .. }) => bitmap, + None => { + error!("the facet value is missing from the facet database: {key:?}"); + return Ok(ControlFlow::Continue(())); + } + }; + let count = search_candidates.intersection_len(&docids); + if count != 0 { + let value = self + .one_original_value_of(fid, &original, docids.min().unwrap())? + .unwrap_or_else(|| query.to_string()); + results.push(FacetValueHit { value, count }); + } + if results.len() >= MAX_NUMBER_OF_FACETS { + return Ok(ControlFlow::Break(())); + } + } + + Ok(ControlFlow::Continue(())) + } } #[derive(Debug, Clone, serde::Serialize, PartialEq)] From 0e2a5951b411c1d8e2e677cefa39587aab87156e Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 24 Jul 2023 18:04:58 +0200 Subject: [PATCH 46/54] Add more advanced tests --- meilisearch/tests/search/facet_search.rs | 48 ++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/meilisearch/tests/search/facet_search.rs b/meilisearch/tests/search/facet_search.rs index 7628f2fed..b38668ad3 100644 --- a/meilisearch/tests/search/facet_search.rs +++ b/meilisearch/tests/search/facet_search.rs @@ -56,6 +56,54 @@ async fn simple_facet_search() { assert_eq!(response["facetHits"].as_array().unwrap().len(), 1); } +#[actix_rt::test] +async fn advanced_facet_search() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.update_settings_filterable_attributes(json!(["genres"])).await; + index.update_settings_typo_tolerance(json!({ "enabled": false })).await; + index.add_documents(documents, None).await; + index.wait_task(2).await; + + let (response, code) = + index.facet_search(json!({"facetName": "genres", "facetQuery": "adventre"})).await; + + assert_eq!(code, 200, "{}", response); + assert_eq!(dbg!(response)["facetHits"].as_array().unwrap().len(), 0); + + let (response, code) = + index.facet_search(json!({"facetName": "genres", "facetQuery": "àdventure"})).await; + + assert_eq!(code, 200, "{}", response); + assert_eq!(response["facetHits"].as_array().unwrap().len(), 1); +} + +#[actix_rt::test] +async fn more_advanced_facet_search() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.update_settings_filterable_attributes(json!(["genres"])).await; + index.update_settings_typo_tolerance(json!({ "disableOnWords": ["adventre"] })).await; + index.add_documents(documents, None).await; + index.wait_task(2).await; + + let (response, code) = + index.facet_search(json!({"facetName": "genres", "facetQuery": "adventre"})).await; + + assert_eq!(code, 200, "{}", response); + assert_eq!(dbg!(response)["facetHits"].as_array().unwrap().len(), 0); + + let (response, code) = + index.facet_search(json!({"facetName": "genres", "facetQuery": "adventure"})).await; + + assert_eq!(code, 200, "{}", response); + assert_eq!(response["facetHits"].as_array().unwrap().len(), 1); +} + #[actix_rt::test] async fn non_filterable_facet_search_error() { let server = Server::new().await; From 86d8bb3a3e0e1e656652cccb345ba6bd7bb6e7d0 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 25 Jul 2023 10:30:50 +0200 Subject: [PATCH 47/54] Make clippy happy (again) --- filter-parser/src/lib.rs | 142 +++++++++++++++++++-------------------- meili-snap/src/lib.rs | 48 ++++++------- 2 files changed, 95 insertions(+), 95 deletions(-) diff --git a/filter-parser/src/lib.rs b/filter-parser/src/lib.rs index 0ae8a1532..11ffbb148 100644 --- a/filter-parser/src/lib.rs +++ b/filter-parser/src/lib.rs @@ -472,6 +472,77 @@ pub fn parse_filter(input: Span) -> IResult { terminated(|input| parse_expression(input, 0), eof)(input) } +impl<'a> std::fmt::Display for FilterCondition<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FilterCondition::Not(filter) => { + write!(f, "NOT ({filter})") + } + FilterCondition::Condition { fid, op } => { + write!(f, "{fid} {op}") + } + FilterCondition::In { fid, els } => { + write!(f, "{fid} IN[")?; + for el in els { + write!(f, "{el}, ")?; + } + write!(f, "]") + } + FilterCondition::Or(els) => { + write!(f, "OR[")?; + for el in els { + write!(f, "{el}, ")?; + } + write!(f, "]") + } + FilterCondition::And(els) => { + write!(f, "AND[")?; + for el in els { + write!(f, "{el}, ")?; + } + write!(f, "]") + } + FilterCondition::GeoLowerThan { point, radius } => { + write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius) + } + FilterCondition::GeoBoundingBox { + top_right_point: top_left_point, + bottom_left_point: bottom_right_point, + } => { + write!( + f, + "_geoBoundingBox([{}, {}], [{}, {}])", + top_left_point[0], + top_left_point[1], + bottom_right_point[0], + bottom_right_point[1] + ) + } + } + } +} +impl<'a> std::fmt::Display for Condition<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Condition::GreaterThan(token) => write!(f, "> {token}"), + Condition::GreaterThanOrEqual(token) => write!(f, ">= {token}"), + Condition::Equal(token) => write!(f, "= {token}"), + Condition::NotEqual(token) => write!(f, "!= {token}"), + Condition::Null => write!(f, "IS NULL"), + Condition::Empty => write!(f, "IS EMPTY"), + Condition::Exists => write!(f, "EXISTS"), + Condition::LowerThan(token) => write!(f, "< {token}"), + Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"), + Condition::Between { from, to } => write!(f, "{from} TO {to}"), + } + } +} +impl<'a> std::fmt::Display for Token<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{{{}}}", self.value()) + } +} + #[cfg(test)] pub mod tests { use super::*; @@ -852,74 +923,3 @@ pub mod tests { assert_eq!(token.value(), s); } } - -impl<'a> std::fmt::Display for FilterCondition<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - FilterCondition::Not(filter) => { - write!(f, "NOT ({filter})") - } - FilterCondition::Condition { fid, op } => { - write!(f, "{fid} {op}") - } - FilterCondition::In { fid, els } => { - write!(f, "{fid} IN[")?; - for el in els { - write!(f, "{el}, ")?; - } - write!(f, "]") - } - FilterCondition::Or(els) => { - write!(f, "OR[")?; - for el in els { - write!(f, "{el}, ")?; - } - write!(f, "]") - } - FilterCondition::And(els) => { - write!(f, "AND[")?; - for el in els { - write!(f, "{el}, ")?; - } - write!(f, "]") - } - FilterCondition::GeoLowerThan { point, radius } => { - write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius) - } - FilterCondition::GeoBoundingBox { - top_right_point: top_left_point, - bottom_left_point: bottom_right_point, - } => { - write!( - f, - "_geoBoundingBox([{}, {}], [{}, {}])", - top_left_point[0], - top_left_point[1], - bottom_right_point[0], - bottom_right_point[1] - ) - } - } - } -} -impl<'a> std::fmt::Display for Condition<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Condition::GreaterThan(token) => write!(f, "> {token}"), - Condition::GreaterThanOrEqual(token) => write!(f, ">= {token}"), - Condition::Equal(token) => write!(f, "= {token}"), - Condition::NotEqual(token) => write!(f, "!= {token}"), - Condition::Null => write!(f, "IS NULL"), - Condition::Empty => write!(f, "IS EMPTY"), - Condition::Exists => write!(f, "EXISTS"), - Condition::LowerThan(token) => write!(f, "< {token}"), - Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"), - Condition::Between { from, to } => write!(f, "{from} TO {to}"), - } - } -} -impl<'a> std::fmt::Display for Token<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{{{}}}", self.value()) - } -} diff --git a/meili-snap/src/lib.rs b/meili-snap/src/lib.rs index dea0d750b..32d3385d9 100644 --- a/meili-snap/src/lib.rs +++ b/meili-snap/src/lib.rs @@ -199,6 +199,30 @@ macro_rules! snapshot { }; } +/// Create a string from the value by serializing it as Json, optionally +/// redacting some parts of it. +/// +/// The second argument to the macro can be an object expression for redaction. +/// It's in the form { selector => replacement }. For more information about redactions +/// refer to the redactions feature in the `insta` guide. +#[macro_export] +macro_rules! json_string { + ($value:expr, {$($k:expr => $v:expr),*$(,)?}) => { + { + let (_, snap) = meili_snap::insta::_prepare_snapshot_for_redaction!($value, {$($k => $v),*}, Json, File); + snap + } + }; + ($value:expr) => {{ + let value = meili_snap::insta::_macro_support::serialize_value( + &$value, + meili_snap::insta::_macro_support::SerializationFormat::Json, + meili_snap::insta::_macro_support::SnapshotLocation::File + ); + value + }}; +} + #[cfg(test)] mod tests { use crate as meili_snap; @@ -250,27 +274,3 @@ mod tests { } } } - -/// Create a string from the value by serializing it as Json, optionally -/// redacting some parts of it. -/// -/// The second argument to the macro can be an object expression for redaction. -/// It's in the form { selector => replacement }. For more information about redactions -/// refer to the redactions feature in the `insta` guide. -#[macro_export] -macro_rules! json_string { - ($value:expr, {$($k:expr => $v:expr),*$(,)?}) => { - { - let (_, snap) = meili_snap::insta::_prepare_snapshot_for_redaction!($value, {$($k => $v),*}, Json, File); - snap - } - }; - ($value:expr) => {{ - let value = meili_snap::insta::_macro_support::serialize_value( - &$value, - meili_snap::insta::_macro_support::SerializationFormat::Json, - meili_snap::insta::_macro_support::SnapshotLocation::File - ); - value - }}; -} From 29ab54b259f94d2403689052e1ba68ffcedceafa Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 25 Jul 2023 12:36:01 +0200 Subject: [PATCH 48/54] Replace the hnsw crate by the instant-distance one --- Cargo.lock | 71 +++++++------------ milli/Cargo.toml | 3 +- milli/src/distance.rs | 40 +++++++---- milli/src/index.rs | 5 +- milli/src/search/new/mod.rs | 49 ++++++------- milli/src/update/delete_documents.rs | 25 ++++--- .../src/update/index_documents/typed_chunk.rs | 54 ++++++++------ 7 files changed, 127 insertions(+), 120 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 370841384..bfc85dda8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,12 +1197,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "dump" version = "1.3.0" @@ -1707,15 +1701,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "hashbrown" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" -dependencies = [ - "ahash 0.7.6", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -1814,22 +1799,6 @@ dependencies = [ "digest", ] -[[package]] -name = "hnsw" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b9740ebf8769ec4ad6762cc951ba18f39bba6dfbc2fbbe46285f7539af79752" -dependencies = [ - "ahash 0.7.6", - "hashbrown 0.11.2", - "libm", - "num-traits", - "rand_core", - "serde", - "smallvec", - "space", -] - [[package]] name = "http" version = "0.2.9" @@ -2008,6 +1977,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "instant-distance" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c619cdaa30bb84088963968bee12a45ea5fbbf355f2c021bcd15589f5ca494a" +dependencies = [ + "num_cpus", + "ordered-float", + "parking_lot", + "rand", + "rayon", + "serde", + "serde-big-array", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -2701,9 +2685,9 @@ dependencies = [ "geoutils", "grenad", "heed", - "hnsw", "indexmap 1.9.3", "insta", + "instant-distance", "itertools", "json-depth-checker", "levenshtein_automata", @@ -2727,7 +2711,6 @@ dependencies = [ "smallstr", "smallvec", "smartstring", - "space", "tempfile", "thiserror", "time", @@ -3607,6 +3590,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + [[package]] name = "serde-cs" version = "0.2.4" @@ -3756,9 +3748,6 @@ name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" -dependencies = [ - "serde", -] [[package]] name = "smartstring" @@ -3781,16 +3770,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "space" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5ab9701ae895386d13db622abf411989deff7109b13b46b6173bb4ce5c1d123" -dependencies = [ - "doc-comment", - "num-traits", -] - [[package]] name = "spin" version = "0.5.2" diff --git a/milli/Cargo.toml b/milli/Cargo.toml index 2689975cd..cbe0794fe 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -33,8 +33,8 @@ heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.6", default-f "lmdb", "sync-read-txn", ] } -hnsw = { version = "0.11.0", features = ["serde1"] } indexmap = { version = "1.9.3", features = ["serde"] } +instant-distance = { version = "0.6.1", features = ["with-serde"] } json-depth-checker = { path = "../json-depth-checker" } levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] } memmap2 = "0.5.10" @@ -48,7 +48,6 @@ rstar = { version = "0.10.0", features = ["serde"] } serde = { version = "1.0.160", features = ["derive"] } serde_json = { version = "1.0.95", features = ["preserve_order"] } slice-group-by = "0.3.0" -space = "0.17.0" smallstr = { version = "0.3.0", features = ["serde"] } smallvec = "1.10.0" smartstring = "1.0.1" diff --git a/milli/src/distance.rs b/milli/src/distance.rs index c838e4bd4..e9e17e647 100644 --- a/milli/src/distance.rs +++ b/milli/src/distance.rs @@ -1,20 +1,36 @@ +use std::ops; + +use instant_distance::Point; use serde::{Deserialize, Serialize}; -use space::Metric; -#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] -pub struct DotProduct; +use crate::normalize_vector; -impl Metric> for DotProduct { - type Unit = u32; +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct NDotProductPoint(Vec); - // Following . - // - // Here is a playground that validate the ordering of the bit representation of floats in range 0.0..=1.0: - // - fn distance(&self, a: &Vec, b: &Vec) -> Self::Unit { - let dist = 1.0 - dot_product_similarity(a, b); +impl NDotProductPoint { + pub fn new(point: Vec) -> Self { + NDotProductPoint(normalize_vector(point)) + } + + pub fn into_inner(self) -> Vec { + self.0 + } +} + +impl ops::Deref for NDotProductPoint { + type Target = [f32]; + + fn deref(&self) -> &Self::Target { + self.0.as_slice() + } +} + +impl Point for NDotProductPoint { + fn distance(&self, other: &Self) -> f32 { + let dist = 1.0 - dot_product_similarity(&self.0, &other.0); debug_assert!(!dist.is_nan()); - dist.to_bits() + dist } } diff --git a/milli/src/index.rs b/milli/src/index.rs index 392ed1705..847ab0088 100644 --- a/milli/src/index.rs +++ b/milli/src/index.rs @@ -8,12 +8,11 @@ use charabia::{Language, Script}; use heed::flags::Flags; use heed::types::*; use heed::{CompactionOption, Database, PolyDatabase, RoTxn, RwTxn}; -use rand_pcg::Pcg32; use roaring::RoaringBitmap; use rstar::RTree; use time::OffsetDateTime; -use crate::distance::DotProduct; +use crate::distance::NDotProductPoint; use crate::error::{InternalError, UserError}; use crate::facet::FacetType; use crate::fields_ids_map::FieldsIdsMap; @@ -31,7 +30,7 @@ use crate::{ }; /// The HNSW data-structure that we serialize, fill and search in. -pub type Hnsw = hnsw::Hnsw, Pcg32, 12, 24>; +pub type Hnsw = instant_distance::Hnsw; pub const DEFAULT_MIN_WORD_LEN_ONE_TYPO: u8 = 5; pub const DEFAULT_MIN_WORD_LEN_TWO_TYPOS: u8 = 9; diff --git a/milli/src/search/new/mod.rs b/milli/src/search/new/mod.rs index c27e02514..ad15d8e91 100644 --- a/milli/src/search/new/mod.rs +++ b/milli/src/search/new/mod.rs @@ -28,7 +28,7 @@ use db_cache::DatabaseCache; use exact_attribute::ExactAttribute; use graph_based_ranking_rule::{Exactness, Fid, Position, Proximity, Typo}; use heed::RoTxn; -use hnsw::Searcher; +use instant_distance::Search; use interner::{DedupInterner, Interner}; pub use logger::visual::VisualSearchLogger; pub use logger::{DefaultSearchLogger, SearchLogger}; @@ -40,19 +40,18 @@ use ranking_rules::{ use resolve_query_graph::{compute_query_graph_docids, PhraseDocIdsCache}; use roaring::RoaringBitmap; use sort::Sort; -use space::Neighbor; use self::distinct::facet_string_values; use self::geo_sort::GeoSort; pub use self::geo_sort::Strategy as GeoSortStrategy; use self::graph_based_ranking_rule::Words; use self::interner::Interned; +use crate::distance::NDotProductPoint; use crate::error::FieldIdMapMissingEntry; use crate::score_details::{ScoreDetails, ScoringStrategy}; use crate::search::new::distinct::apply_distinct_rule; use crate::{ - normalize_vector, AscDesc, DocumentId, Filter, Index, Member, Result, TermsMatchingStrategy, - UserError, BEU32, + AscDesc, DocumentId, Filter, Index, Member, Result, TermsMatchingStrategy, UserError, BEU32, }; /// A structure used throughout the execution of a search query. @@ -445,29 +444,31 @@ pub fn execute_search( check_sort_criteria(ctx, sort_criteria.as_ref())?; if let Some(vector) = vector { - let mut searcher = Searcher::new(); - let hnsw = ctx.index.vector_hnsw(ctx.txn)?.unwrap_or_default(); - let ef = hnsw.len().min(100); - let mut dest = vec![Neighbor { index: 0, distance: 0 }; ef]; - let vector = normalize_vector(vector.clone()); - let neighbors = hnsw.nearest(&vector, ef, &mut searcher, &mut dest[..]); + let mut search = Search::default(); + let docids = match ctx.index.vector_hnsw(ctx.txn)? { + Some(hnsw) => { + let vector = NDotProductPoint::new(vector.clone()); + let neighbors = hnsw.search(&vector, &mut search); - let mut docids = Vec::new(); - let mut uniq_docids = RoaringBitmap::new(); - for Neighbor { index, distance: _ } in neighbors.iter() { - let index = BEU32::new(*index as u32); - let docid = ctx.index.vector_id_docid.get(ctx.txn, &index)?.unwrap().get(); - if universe.contains(docid) && uniq_docids.insert(docid) { - docids.push(docid); - if docids.len() == (from + length) { - break; + let mut docids = Vec::new(); + let mut uniq_docids = RoaringBitmap::new(); + for instant_distance::Item { distance: _, pid, point: _ } in neighbors { + let index = BEU32::new(pid.into_inner()); + let docid = ctx.index.vector_id_docid.get(ctx.txn, &index)?.unwrap().get(); + if universe.contains(docid) && uniq_docids.insert(docid) { + docids.push(docid); + if docids.len() == (from + length) { + break; + } + } } - } - } - // return the nearest documents that are also part of the candidates - // along with a dummy list of scores that are useless in this context. - let docids: Vec<_> = docids.into_iter().skip(from).take(length).collect(); + // return the nearest documents that are also part of the candidates + // along with a dummy list of scores that are useless in this context. + docids.into_iter().skip(from).take(length).collect() + } + None => Vec::new(), + }; return Ok(PartialSearchResult { candidates: universe, diff --git a/milli/src/update/delete_documents.rs b/milli/src/update/delete_documents.rs index c9124e591..906d6922f 100644 --- a/milli/src/update/delete_documents.rs +++ b/milli/src/update/delete_documents.rs @@ -4,10 +4,9 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use fst::IntoStreamer; use heed::types::{ByteSlice, DecodeIgnore, Str, UnalignedSlice}; use heed::{BytesDecode, BytesEncode, Database, RwIter}; -use hnsw::Searcher; +use instant_distance::PointId; use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; -use space::KnnPoints; use time::OffsetDateTime; use super::facet::delete::FacetsDelete; @@ -436,24 +435,24 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { // An ugly and slow way to remove the vectors from the HNSW // It basically reconstructs the HNSW from scratch without editing the current one. - let current_hnsw = self.index.vector_hnsw(self.wtxn)?.unwrap_or_default(); - if !current_hnsw.is_empty() { - let mut new_hnsw = Hnsw::default(); - let mut searcher = Searcher::new(); - let mut new_vector_id_docids = Vec::new(); - + if let Some(current_hnsw) = self.index.vector_hnsw(self.wtxn)? { + let mut points = Vec::new(); + let mut docids = Vec::new(); for result in vector_id_docid.iter(self.wtxn)? { let (vector_id, docid) = result?; if !self.to_delete_docids.contains(docid.get()) { - let vector = current_hnsw.get_point(vector_id.get() as usize).clone(); - let vector_id = new_hnsw.insert(vector, &mut searcher); - new_vector_id_docids.push((vector_id as u32, docid)); + let pid = PointId::from(vector_id.get()); + let vector = current_hnsw[pid].clone(); + points.push(vector); + docids.push(docid); } } + let (new_hnsw, pids) = Hnsw::builder().build_hnsw(points); + vector_id_docid.clear(self.wtxn)?; - for (vector_id, docid) in new_vector_id_docids { - vector_id_docid.put(self.wtxn, &BEU32::new(vector_id), &docid)?; + for (pid, docid) in pids.into_iter().zip(docids) { + vector_id_docid.put(self.wtxn, &BEU32::new(pid.into_inner()), &docid)?; } self.index.put_vector_hnsw(self.wtxn, &new_hnsw)?; } diff --git a/milli/src/update/index_documents/typed_chunk.rs b/milli/src/update/index_documents/typed_chunk.rs index 3f197fbd1..921ce4ecd 100644 --- a/milli/src/update/index_documents/typed_chunk.rs +++ b/milli/src/update/index_documents/typed_chunk.rs @@ -9,22 +9,19 @@ use charabia::{Language, Script}; use grenad::MergerBuilder; use heed::types::ByteSlice; use heed::RwTxn; -use hnsw::Searcher; use roaring::RoaringBitmap; -use space::KnnPoints; use super::helpers::{ self, merge_ignore_values, serialize_roaring_bitmap, valid_lmdb_key, CursorClonableMmap, }; use super::{ClonableMmap, MergeFn}; +use crate::distance::NDotProductPoint; use crate::error::UserError; use crate::facet::FacetType; +use crate::index::Hnsw; use crate::update::facet::FacetsUpdate; use crate::update::index_documents::helpers::{as_cloneable_grenad, try_split_array_at}; -use crate::{ - lat_lng_to_xyz, normalize_vector, CboRoaringBitmapCodec, DocumentId, GeoPoint, Index, Result, - BEU32, -}; +use crate::{lat_lng_to_xyz, CboRoaringBitmapCodec, DocumentId, GeoPoint, Index, Result, BEU32}; pub(crate) enum TypedChunk { FieldIdDocidFacetStrings(grenad::Reader), @@ -230,17 +227,20 @@ pub(crate) fn write_typed_chunk_into_index( index.put_geo_faceted_documents_ids(wtxn, &geo_faceted_docids)?; } TypedChunk::VectorPoints(vector_points) => { - let mut hnsw = index.vector_hnsw(wtxn)?.unwrap_or_default(); - let mut searcher = Searcher::new(); - - let mut expected_dimensions = match index.vector_id_docid.iter(wtxn)?.next() { - Some(result) => { - let (vector_id, _) = result?; - Some(hnsw.get_point(vector_id.get() as usize).len()) - } - None => None, + let (pids, mut points): (Vec<_>, Vec<_>) = match index.vector_hnsw(wtxn)? { + Some(hnsw) => hnsw.iter().map(|(pid, point)| (pid, point.clone())).unzip(), + None => Default::default(), }; + // Convert the PointIds into DocumentIds + let mut docids = Vec::new(); + for pid in pids { + let docid = + index.vector_id_docid.get(wtxn, &BEU32::new(pid.into_inner()))?.unwrap(); + docids.push(docid.get()); + } + + let mut expected_dimensions = points.get(0).map(|p| p.len()); let mut cursor = vector_points.into_cursor()?; while let Some((key, value)) = cursor.move_on_next()? { // convert the key back to a u32 (4 bytes) @@ -256,12 +256,26 @@ pub(crate) fn write_typed_chunk_into_index( return Err(UserError::InvalidVectorDimensions { expected, found })?; } - let vector = normalize_vector(vector); - let vector_id = hnsw.insert(vector, &mut searcher) as u32; - index.vector_id_docid.put(wtxn, &BEU32::new(vector_id), &BEU32::new(docid))?; + points.push(NDotProductPoint::new(vector)); + docids.push(docid); } - log::debug!("There are {} entries in the HNSW so far", hnsw.len()); - index.put_vector_hnsw(wtxn, &hnsw)?; + + assert_eq!(docids.len(), points.len()); + + let hnsw_length = points.len(); + let (new_hnsw, pids) = Hnsw::builder().build_hnsw(points); + + index.vector_id_docid.clear(wtxn)?; + for (docid, pid) in docids.into_iter().zip(pids) { + index.vector_id_docid.put( + wtxn, + &BEU32::new(pid.into_inner()), + &BEU32::new(docid), + )?; + } + + log::debug!("There are {} entries in the HNSW so far", hnsw_length); + index.put_vector_hnsw(wtxn, &new_hnsw)?; } TypedChunk::ScriptLanguageDocids(hash_pair) => { let mut buffer = Vec::new(); From 59201a7852a1a160165cba9a377e990d021f452a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 25 Jul 2023 14:46:23 +0200 Subject: [PATCH 49/54] Use snapshot instead of asserts Co-authored-by: Many the fish --- meilisearch/tests/search/facet_search.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/meilisearch/tests/search/facet_search.rs b/meilisearch/tests/search/facet_search.rs index b38668ad3..446396856 100644 --- a/meilisearch/tests/search/facet_search.rs +++ b/meilisearch/tests/search/facet_search.rs @@ -1,3 +1,4 @@ +use meili_snap::snapshot; use once_cell::sync::Lazy; use serde_json::{json, Value}; @@ -70,14 +71,14 @@ async fn advanced_facet_search() { let (response, code) = index.facet_search(json!({"facetName": "genres", "facetQuery": "adventre"})).await; - assert_eq!(code, 200, "{}", response); - assert_eq!(dbg!(response)["facetHits"].as_array().unwrap().len(), 0); + snapshot!(code, @"200 OK"); + snapshot!(response["facetHits"].as_array().unwrap().len(), @"0"); let (response, code) = index.facet_search(json!({"facetName": "genres", "facetQuery": "àdventure"})).await; - assert_eq!(code, 200, "{}", response); - assert_eq!(response["facetHits"].as_array().unwrap().len(), 1); + snapshot!(code, @"200 OK"); + snapshot!(response["facetHits"].as_array().unwrap().len(), @"1"); } #[actix_rt::test] @@ -94,14 +95,14 @@ async fn more_advanced_facet_search() { let (response, code) = index.facet_search(json!({"facetName": "genres", "facetQuery": "adventre"})).await; - assert_eq!(code, 200, "{}", response); - assert_eq!(dbg!(response)["facetHits"].as_array().unwrap().len(), 0); + snapshot!(code, @"200 OK"); + snapshot!(response["facetHits"].as_array().unwrap().len(), @"0"); let (response, code) = index.facet_search(json!({"facetName": "genres", "facetQuery": "adventure"})).await; - assert_eq!(code, 200, "{}", response); - assert_eq!(response["facetHits"].as_array().unwrap().len(), 1); + snapshot!(code, @"200 OK"); + snapshot!(response["facetHits"].as_array().unwrap().len(), @"1"); } #[actix_rt::test] From 88559a2d54a817b1dda32bbe13b79f28ce2a036d Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Tue, 25 Jul 2023 15:49:33 +0200 Subject: [PATCH 50/54] Fix score details casing --- meilisearch/tests/search/mod.rs | 4 ++-- milli/src/score_details.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/meilisearch/tests/search/mod.rs b/meilisearch/tests/search/mod.rs index d560ba734..6fcc33309 100644 --- a/meilisearch/tests/search/mod.rs +++ b/meilisearch/tests/search/mod.rs @@ -820,8 +820,8 @@ async fn experimental_feature_score_details() { }, "attribute": { "order": 3, - "attribute_ranking_order_score": 1.0, - "query_word_distance_score": 0.8095238095238095, + "attributeRankingOrderScore": 1.0, + "queryWordDistanceScore": 0.8095238095238095, "score": 0.9365079365079364 }, "exactness": { diff --git a/milli/src/score_details.rs b/milli/src/score_details.rs index 0235436d8..8fc998ae4 100644 --- a/milli/src/score_details.rs +++ b/milli/src/score_details.rs @@ -84,7 +84,7 @@ impl ScoreDetails { // For now, fid is a virtual rule always followed by the "position" rule let fid_details = serde_json::json!({ "order": order, - "attribute_ranking_order_score": fid.local_score(), + "attributeRankingOrderScore": fid.local_score(), }); details_map.insert("attribute".into(), fid_details); order += 1; @@ -102,7 +102,7 @@ impl ScoreDetails { }; attribute_details - .insert("query_word_distance_score".into(), position.local_score().into()); + .insert("queryWordDistanceScore".into(), position.local_score().into()); let score = Rank::global_score([fid_details, *position].iter().copied()); attribute_details.insert("score".into(), score.into()); From d8b47b689edbdae1c98c9f22c3c23bd959edad59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 26 Jul 2023 13:52:37 +0200 Subject: [PATCH 51/54] Use the new read-txn-no-tls heed feature --- Cargo.lock | 8 ++++---- milli/Cargo.toml | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bfc85dda8..4aa7c7b67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1737,8 +1737,8 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "heed" -version = "0.12.5" -source = "git+https://github.com/meilisearch/heed?tag=v0.12.6#8c5b94225fc949c02bb7b900cc50ffaf6b584b1e" +version = "0.12.7" +source = "git+https://github.com/meilisearch/heed?tag=v0.12.7#061a5276b1f336f5f3302bee291e336041d88632" dependencies = [ "byteorder", "heed-traits", @@ -1755,12 +1755,12 @@ dependencies = [ [[package]] name = "heed-traits" version = "0.7.0" -source = "git+https://github.com/meilisearch/heed?tag=v0.12.6#8c5b94225fc949c02bb7b900cc50ffaf6b584b1e" +source = "git+https://github.com/meilisearch/heed?tag=v0.12.7#061a5276b1f336f5f3302bee291e336041d88632" [[package]] name = "heed-types" version = "0.7.2" -source = "git+https://github.com/meilisearch/heed?tag=v0.12.6#8c5b94225fc949c02bb7b900cc50ffaf6b584b1e" +source = "git+https://github.com/meilisearch/heed?tag=v0.12.7#061a5276b1f336f5f3302bee291e336041d88632" dependencies = [ "bincode", "heed-traits", diff --git a/milli/Cargo.toml b/milli/Cargo.toml index cbe0794fe..4831ea990 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -29,9 +29,8 @@ geoutils = "0.5.1" grenad = { version = "0.4.4", default-features = false, features = [ "tempfile", ] } -heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.6", default-features = false, features = [ - "lmdb", - "sync-read-txn", +heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.7", default-features = false, features = [ + "lmdb", "read-txn-no-tls" ] } indexmap = { version = "1.9.3", features = ["serde"] } instant-distance = { version = "0.6.1", features = ["with-serde"] } From fae61372be4f800ec2cd6675313955c109d22038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa?= Date: Wed, 26 Jul 2023 15:54:43 +0200 Subject: [PATCH 52/54] Redirect CTAs to Cloud landing page --- README.md | 2 +- meilisearch/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e05860fbe..ad0bb69c2 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ You may also want to check out [Meilisearch 101](https://www.meilisearch.com/doc ## ⚡ Supercharge your Meilisearch experience -Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=engine&utm_medium=meilisearch). No credit card required. +Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). No credit card required. ## 🧰 SDKs & integration tools diff --git a/meilisearch/src/main.rs b/meilisearch/src/main.rs index b0ca9dd7b..246d62c3b 100644 --- a/meilisearch/src/main.rs +++ b/meilisearch/src/main.rs @@ -187,7 +187,7 @@ Anonymous telemetry:\t\"Enabled\"" } eprintln!(); - eprintln!("Check out Meilisearch Cloud!\thttps://cloud.meilisearch.com/login?utm_campaign=oss&utm_source=engine&utm_medium=cli"); + eprintln!("Check out Meilisearch Cloud!\thttps://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=engine&utm_medium=cli"); eprintln!("Documentation:\t\t\thttps://www.meilisearch.com/docs"); eprintln!("Source code:\t\t\thttps://github.com/meilisearch/meilisearch"); eprintln!("Discord:\t\t\thttps://discord.meilisearch.com"); From 3b9a87c7908ce59abb1ae7014fe67065f2343e61 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 27 Jul 2023 13:16:32 +0200 Subject: [PATCH 53/54] Update mini-dashboard to version 0.2.11 --- meilisearch/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index b048c9720..24a6fbd0f 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -141,5 +141,5 @@ thai = ["meilisearch-types/thai"] greek = ["meilisearch-types/greek"] [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.9/build.zip" -sha1 = "9c713298788363e41087f2b40e7509d390033e30" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.11/build.zip" +sha1 = "83cd44ed1e5f97ecb581dc9f958a63f4ccc982d9" From fcdd20b53367385eda59bdc0c228ff361c5e9798 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 1 Aug 2023 16:06:33 +0200 Subject: [PATCH 54/54] Fix README after git conflict --- README.md | 68 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index ad0bb69c2..88621729d 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,20 @@

- - + + + + + +

- Website | + Website | Roadmap | - Meilisearch Cloud | - Blog | - Documentation | - FAQ | - Discord + Meilisearch Cloud | + Blog | + Documentation | + FAQ | + Discord

@@ -24,40 +28,40 @@ Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.

- + A bright colored application for finding movies screening near the user - + A dark colored application for finding movies screening near the user

-🔥 [**Try it!**](https://where2watch.meilisearch.com/) 🔥 +🔥 [**Try it!**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-link) 🔥 ## ✨ Features - **Search-as-you-type:** find search results in less than 50 milliseconds -- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy#typo-tolerance):** get relevant matches even when queries contain typos and misspellings -- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code -- **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting):** sort results based on price, date, or pretty much anything else your users need -- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy#synonyms):** configure synonyms to include more relevant content in your search results -- **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch):** filter and sort documents based on geographic data -- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet -- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys):** control which users can access what data with API keys that allow fine-grained permissions handling -- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens):** personalize search results for any number of application tenants +- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#typo-tolerance):** get relevant matches even when queries contain typos and misspellings +- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code +- **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need +- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#synonyms):** configure synonyms to include more relevant content in your search results +- **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data +- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet +- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling +- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants - **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets -- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview):** integrate Meilisearch in your technical stack with our plugins and SDKs +- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs - **Easy to install, deploy, and maintain** ## 📖 Documentation -You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/). +You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs). ## 🚀 Getting started -For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start) guide. +For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide. -You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting) for an introduction to some of Meilisearch's most popular features. +You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) for an introduction to some of Meilisearch's most popular features. ## ⚡ Supercharge your Meilisearch experience @@ -67,29 +71,29 @@ Say goodbye to server deployment and manual updates with [Meilisearch Cloud](htt Install one of our SDKs in your project for seamless integration between Meilisearch and your favorite language or framework! -Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks). +Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-link). -[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks) +[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-logos) ## ⚙️ Advanced usage -Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview) close at hand. +Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) close at hand. -We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering), [sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting), [geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens). +We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced). -Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes). +Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced). ## 📊 Telemetry -Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry#how-to-disable-data-collection) whenever you want. +Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want. -To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data. +To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data. -If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry) of our documentation. +If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation. ## 📫 Get in touch! -Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/) +Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact) 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.