From fc2590fc9dfdf283a15851d367a4b30bcf09aee6 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Tue, 8 Aug 2023 16:43:08 +0200 Subject: [PATCH 01/20] Add a test --- meilisearch/tests/search/mod.rs | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/meilisearch/tests/search/mod.rs b/meilisearch/tests/search/mod.rs index 6fcc33309..3aefe7e83 100644 --- a/meilisearch/tests/search/mod.rs +++ b/meilisearch/tests/search/mod.rs @@ -1104,3 +1104,59 @@ async fn camelcased_words() { }) .await; } + +#[actix_rt::test] +async fn simple_search_with_strange_synonyms() { + let server = Server::new().await; + let index = server.index("test"); + + index.update_settings(json!({ "synonyms": {"&": ["to"], "to": ["&"]} })).await; + let r = index.wait_task(0).await; + meili_snap::snapshot!(r["status"], @r###""succeeded""###); + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_task(1).await; + + index + .search(json!({"q": "How to train"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "title": "How to Train Your Dragon: The Hidden World", + "id": "166428" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "How & train"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "title": "How to Train Your Dragon: The Hidden World", + "id": "166428" + } + ] + "###); + }) + .await; + + index + .search(json!({"q": "to"}), |response, code| { + meili_snap::snapshot!(code, @"200 OK"); + meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###" + [ + { + "title": "How to Train Your Dragon: The Hidden World", + "id": "166428" + } + ] + "###); + }) + .await; +} From 8dc5acf998a3f7caf0126983bfa45265e7efed94 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Tue, 8 Aug 2023 16:52:36 +0200 Subject: [PATCH 02/20] Try fix --- milli/src/update/settings.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/milli/src/update/settings.rs b/milli/src/update/settings.rs index 33f86a4bb..d3fdac0c7 100644 --- a/milli/src/update/settings.rs +++ b/milli/src/update/settings.rs @@ -477,13 +477,18 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { for (word, synonyms) in synonyms { // Normalize both the word and associated synonyms. let normalized_word = normalize(&tokenizer, word); - let normalized_synonyms = - synonyms.iter().map(|synonym| normalize(&tokenizer, synonym)); + let normalized_synonyms: Vec<_> = synonyms + .iter() + .map(|synonym| normalize(&tokenizer, synonym)) + .filter(|synonym| !synonym.is_empty()) + .collect(); // Store the normalized synonyms under the normalized word, // merging the possible duplicate words. - let entry = new_synonyms.entry(normalized_word).or_insert_with(Vec::new); - entry.extend(normalized_synonyms); + if !normalized_word.is_empty() && !normalized_synonyms.is_empty() { + let entry = new_synonyms.entry(normalized_word).or_insert_with(Vec::new); + entry.extend(normalized_synonyms.into_iter()); + } } // Make sure that we don't have duplicate synonyms. From 6db80b0836e2ff7144cefaec80bc9b49c3d0fcc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 24 Aug 2023 11:24:47 +0200 Subject: [PATCH 03/20] Define the full Homebrew formula path --- .github/workflows/publish-apt-brew-pkg.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-apt-brew-pkg.yml b/.github/workflows/publish-apt-brew-pkg.yml index 043715224..452776e38 100644 --- a/.github/workflows/publish-apt-brew-pkg.yml +++ b/.github/workflows/publish-apt-brew-pkg.yml @@ -53,5 +53,6 @@ jobs: uses: mislav/bump-homebrew-formula-action@v2 with: formula-name: meilisearch + formula-path: Formula/m/meilisearch.rb env: COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }} From 085aad0a94b6431e72849f4807181a5015994ff7 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 4 Sep 2023 14:39:33 +0200 Subject: [PATCH 04/20] Add a test --- milli/src/update/settings.rs | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/milli/src/update/settings.rs b/milli/src/update/settings.rs index 7e52c04c1..023e09aa0 100644 --- a/milli/src/update/settings.rs +++ b/milli/src/update/settings.rs @@ -1422,6 +1422,43 @@ mod tests { assert!(result.documents_ids.is_empty()); } + #[test] + fn thai_synonyms() { + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; + + let mut wtxn = index.write_txn().unwrap(); + // Send 3 documents with ids from 1 to 3. + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "name": "ยี่ปุ่น" }, + { "name": "ญี่ปุ่น" }, + ]), + ) + .unwrap(); + + // In the same transaction provide some synonyms + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_synonyms(btreemap! { + "japanese".to_string() => vec!["ญี่ปุ่น", "ยี่ปุ่น"], + }); + }) + .unwrap(); + wtxn.commit().unwrap(); + + // Ensure synonyms are effectively stored + let rtxn = index.read_txn().unwrap(); + let synonyms = index.synonyms(&rtxn).unwrap(); + assert!(!synonyms.is_empty()); // at this point the index should return something + + // Check that we can use synonyms + let result = index.search(&rtxn).query("japanese").execute().unwrap(); + assert_eq!(result.documents_ids.len(), 2); + } + #[test] fn setting_searchable_recomputes_other_settings() { let index = TempIndex::new(); From 8ac5b765bc0d22dd8448e671c66529ca668e3389 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 4 Sep 2023 14:39:52 +0200 Subject: [PATCH 05/20] Fix synonyms normalization --- milli/src/update/settings.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/milli/src/update/settings.rs b/milli/src/update/settings.rs index 023e09aa0..b0452315d 100644 --- a/milli/src/update/settings.rs +++ b/milli/src/update/settings.rs @@ -573,7 +573,7 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { tokenizer .tokenize(text) .filter_map(|token| { - if token.is_word() { + if token.is_word() && !token.lemma().is_empty() { Some(token.lemma().to_string()) } else { None @@ -1443,7 +1443,7 @@ mod tests { index .update_settings_using_wtxn(&mut wtxn, |settings| { settings.set_synonyms(btreemap! { - "japanese".to_string() => vec!["ญี่ปุ่น", "ยี่ปุ่น"], + "japanese".to_string() => vec![S("ญี่ปุ่น"), S("ยี่ปุ่น")], }); }) .unwrap(); From 66aa6d5871050ea0c18bd672dba83681ded8869a Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Tue, 5 Sep 2023 15:44:14 +0200 Subject: [PATCH 06/20] Ignore tokens with empty normalized value during indexing process --- .../index_documents/extract/extract_docid_word_positions.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/milli/src/update/index_documents/extract/extract_docid_word_positions.rs b/milli/src/update/index_documents/extract/extract_docid_word_positions.rs index ac041a8b0..1c24a0fcf 100644 --- a/milli/src/update/index_documents/extract/extract_docid_word_positions.rs +++ b/milli/src/update/index_documents/extract/extract_docid_word_positions.rs @@ -226,9 +226,9 @@ fn process_tokens<'a>( ) -> impl Iterator)> { tokens .skip_while(|token| token.is_separator()) - .scan((0, None), |(offset, prev_kind), token| { + .scan((0, None), |(offset, prev_kind), mut token| { match token.kind { - TokenKind::Word | TokenKind::StopWord | TokenKind::Unknown => { + TokenKind::Word | TokenKind::StopWord if !token.lemma().is_empty() => { *offset += match *prev_kind { Some(TokenKind::Separator(SeparatorKind::Hard)) => 8, Some(_) => 1, @@ -244,7 +244,7 @@ fn process_tokens<'a>( { *prev_kind = Some(token.kind); } - _ => (), + _ => token.kind = TokenKind::Unknown, } Some((*offset, token)) }) From 93285041a93725eec7fd97f7f31554c6c17459bb Mon Sep 17 00:00:00 2001 From: curquiza Date: Wed, 6 Sep 2023 09:23:20 +0000 Subject: [PATCH 07/20] Update version for the next release (v1.3.3) in Cargo.toml --- Cargo.lock | 28 ++++++++++++++-------------- Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ab4e53d0..c1392f4a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -469,7 +469,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "benchmarks" -version = "1.3.2" +version = "1.3.3" dependencies = [ "anyhow", "bytes", @@ -1199,7 +1199,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.3.2" +version = "1.3.3" dependencies = [ "anyhow", "big_s", @@ -1413,7 +1413,7 @@ dependencies = [ [[package]] name = "file-store" -version = "1.3.2" +version = "1.3.3" dependencies = [ "faux", "tempfile", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.3.2" +version = "1.3.3" dependencies = [ "insta", "nom", @@ -1454,7 +1454,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.3.2" +version = "1.3.3" dependencies = [ "criterion", "serde_json", @@ -1572,7 +1572,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.3.2" +version = "1.3.3" dependencies = [ "arbitrary", "clap", @@ -1894,7 +1894,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "1.3.2" +version = "1.3.3" dependencies = [ "anyhow", "big_s", @@ -2081,7 +2081,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.3.2" +version = "1.3.3" dependencies = [ "criterion", "serde_json", @@ -2493,7 +2493,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.3.2" +version = "1.3.3" dependencies = [ "insta", "md5", @@ -2502,7 +2502,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.3.2" +version = "1.3.3" dependencies = [ "actix-cors", "actix-http", @@ -2591,7 +2591,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.3.2" +version = "1.3.3" dependencies = [ "base64 0.21.2", "enum-iterator", @@ -2610,7 +2610,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.3.2" +version = "1.3.3" dependencies = [ "actix-web", "anyhow", @@ -2664,7 +2664,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.3.2" +version = "1.3.3" dependencies = [ "big_s", "bimap", @@ -2994,7 +2994,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "permissive-json-pointer" -version = "1.3.2" +version = "1.3.3" dependencies = [ "big_s", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 98b484ee4..afa79af5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ members = [ ] [workspace.package] -version = "1.3.2" +version = "1.3.3" authors = ["Quentin de Quelen ", "Clément Renault "] description = "Meilisearch HTTP server" homepage = "https://meilisearch.com" From e02d0064bd620b02e9d00772189c9fd772aa8cfa Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 6 Sep 2023 11:49:27 +0200 Subject: [PATCH 08/20] Add a test case scenario --- milli/src/update/index_documents/mod.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 849e84035..3704faf44 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -2519,6 +2519,25 @@ mod tests { db_snap!(index, word_position_docids, 3, @"74f556b91d161d997a89468b4da1cb8f"); } + /// Index multiple different number of vectors in documents. + /// Vectors must be of the same length. + #[test] + fn test_multiple_vectors() { + let index = TempIndex::new(); + + index.add_documents(documents!([{"id": 0, "_vectors": [[0, 1, 2], [3, 4, 5]] }])).unwrap(); + index.add_documents(documents!([{"id": 1, "_vectors": [6, 7, 8] }])).unwrap(); + index + .add_documents( + documents!([{"id": 2, "_vectors": [[9, 10, 11], [12, 13, 14], [15, 16, 17]] }]), + ) + .unwrap(); + + let rtxn = index.read_txn().unwrap(); + let res = index.search(&rtxn).vector([0.0, 1.0, 2.0]).execute().unwrap(); + assert_eq!(res.documents_ids.len(), 3); + } + #[test] fn reproduce_the_bug() { /* From 679c0b0f970c111b3c86309354bb82ce6be47e3a Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 6 Sep 2023 12:20:25 +0200 Subject: [PATCH 09/20] Extract the vectors from the non-flattened version of the documents --- .../src/update/index_documents/extract/mod.rs | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/milli/src/update/index_documents/extract/mod.rs b/milli/src/update/index_documents/extract/mod.rs index 6259c7272..6a3d6d972 100644 --- a/milli/src/update/index_documents/extract/mod.rs +++ b/milli/src/update/index_documents/extract/mod.rs @@ -55,7 +55,13 @@ pub(crate) fn data_from_obkv_documents( original_obkv_chunks .par_bridge() .map(|original_documents_chunk| { - send_original_documents_data(original_documents_chunk, lmdb_writer_sx.clone()) + send_original_documents_data( + original_documents_chunk, + indexer, + lmdb_writer_sx.clone(), + vectors_field_id, + primary_key_id, + ) }) .collect::>()?; @@ -72,7 +78,6 @@ pub(crate) fn data_from_obkv_documents( &faceted_fields, primary_key_id, geo_fields_ids, - vectors_field_id, &stop_words, max_positions_per_attributes, ) @@ -257,11 +262,33 @@ fn spawn_extraction_task( /// - documents fn send_original_documents_data( original_documents_chunk: Result>, + indexer: GrenadParameters, lmdb_writer_sx: Sender>, + vectors_field_id: Option, + primary_key_id: FieldId, ) -> Result<()> { let original_documents_chunk = original_documents_chunk.and_then(|c| unsafe { as_cloneable_grenad(&c) })?; + if let Some(vectors_field_id) = vectors_field_id { + let documents_chunk_cloned = original_documents_chunk.clone(); + let lmdb_writer_sx_cloned = lmdb_writer_sx.clone(); + rayon::spawn(move || { + let result = extract_vector_points( + documents_chunk_cloned, + indexer, + primary_key_id, + vectors_field_id, + ); + let _ = match result { + Ok(vector_points) => { + lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints(vector_points))) + } + Err(error) => lmdb_writer_sx_cloned.send(Err(error)), + }; + }); + } + // TODO: create a custom internal error lmdb_writer_sx.send(Ok(TypedChunk::Documents(original_documents_chunk))).unwrap(); Ok(()) @@ -283,7 +310,6 @@ fn send_and_extract_flattened_documents_data( faceted_fields: &HashSet, primary_key_id: FieldId, geo_fields_ids: Option<(FieldId, FieldId)>, - vectors_field_id: Option, stop_words: &Option>, max_positions_per_attributes: Option, ) -> Result<( @@ -312,25 +338,6 @@ fn send_and_extract_flattened_documents_data( }); } - if let Some(vectors_field_id) = vectors_field_id { - let documents_chunk_cloned = flattened_documents_chunk.clone(); - let lmdb_writer_sx_cloned = lmdb_writer_sx.clone(); - rayon::spawn(move || { - let result = extract_vector_points( - documents_chunk_cloned, - indexer, - primary_key_id, - vectors_field_id, - ); - let _ = match result { - Ok(vector_points) => { - lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints(vector_points))) - } - Err(error) => lmdb_writer_sx_cloned.send(Err(error)), - }; - }); - } - let (docid_word_positions_chunk, docid_fid_facet_values_chunks): (Result<_>, Result<_>) = rayon::join( || { From b42d48187a84e3e8acd7e337ccd5755967a2ecbe Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 6 Sep 2023 11:39:06 +0200 Subject: [PATCH 10/20] Add a test case scenario --- filter-parser/src/lib.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/filter-parser/src/lib.rs b/filter-parser/src/lib.rs index 11ffbb148..5760c8865 100644 --- a/filter-parser/src/lib.rs +++ b/filter-parser/src/lib.rs @@ -545,6 +545,8 @@ impl<'a> std::fmt::Display for Token<'a> { #[cfg(test)] pub mod tests { + use FilterCondition as Fc; + use super::*; /// Create a raw [Token]. You must specify the string that appear BEFORE your element followed by your element @@ -556,14 +558,22 @@ pub mod tests { unsafe { Span::new_from_raw_offset(offset, lines as u32, value, "") }.into() } + fn p(s: &str) -> impl std::fmt::Display + '_ { + Fc::parse(s).unwrap().unwrap() + } + + #[test] + fn parse_escaped() { + insta::assert_display_snapshot!(p(r#"title = 'foo\\'"#), @r#"{title} = {foo\}"#); + insta::assert_display_snapshot!(p(r#"title = 'foo\\\\'"#), @r#"{title} = {foo\\}"#); + insta::assert_display_snapshot!(p(r#"title = 'foo\\\\\\'"#), @r#"{title} = {foo\\\}"#); + insta::assert_display_snapshot!(p(r#"title = 'foo\\\\\\\\'"#), @r#"{title} = {foo\\\\}"#); + // but it also works with other sequencies + insta::assert_display_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}"); + } + #[test] fn parse() { - use FilterCondition as Fc; - - fn p(s: &str) -> impl std::fmt::Display + '_ { - Fc::parse(s).unwrap().unwrap() - } - // Test equal insta::assert_display_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}"); insta::assert_display_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}"); From ea7806091639da9d702550c9940e63dad2da4795 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 6 Sep 2023 11:39:36 +0200 Subject: [PATCH 11/20] Fix tests that were supposed to escape characters --- filter-parser/src/value.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/filter-parser/src/value.rs b/filter-parser/src/value.rs index 735352fc3..932aabca9 100644 --- a/filter-parser/src/value.rs +++ b/filter-parser/src/value.rs @@ -318,17 +318,17 @@ pub mod test { ("\"cha'nnel\"", "cha'nnel", false), ("I'm tamo", "I", false), // escaped thing but not quote - (r#""\\""#, r#"\\"#, false), - (r#""\\\\\\""#, r#"\\\\\\"#, false), - (r#""aa\\aa""#, r#"aa\\aa"#, false), + (r#""\\""#, r#"\"#, true), + (r#""\\\\\\""#, r#"\\\"#, true), + (r#""aa\\aa""#, r#"aa\aa"#, true), // with double quote (r#""Hello \"world\"""#, r#"Hello "world""#, true), - (r#""Hello \\\"world\\\"""#, r#"Hello \\"world\\""#, true), + (r#""Hello \\\"world\\\"""#, r#"Hello \"world\""#, true), (r#""I'm \"super\" tamo""#, r#"I'm "super" tamo"#, true), (r#""\"\"""#, r#""""#, true), // with simple quote (r#"'Hello \'world\''"#, r#"Hello 'world'"#, true), - (r#"'Hello \\\'world\\\''"#, r#"Hello \\'world\\'"#, true), + (r#"'Hello \\\'world\\\''"#, r#"Hello \'world\'"#, true), (r#"'I\'m "super" tamo'"#, r#"I'm "super" tamo"#, true), (r#"'\'\''"#, r#"''"#, true), ]; @@ -350,7 +350,14 @@ pub mod test { "Filter `{}` was not supposed to be escaped", input ); - assert_eq!(token.value(), expected, "Filter `{}` failed.", input); + assert_eq!( + token.value(), + expected, + "Filter `{}` failed by giving `{}` instead of `{}`.", + input, + token.value(), + expected + ); } } From 03d0f628bdf6ba9cbe29d427c6001d64b253e2c0 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 6 Sep 2023 11:40:21 +0200 Subject: [PATCH 12/20] Use the unescaper crate to unescape any char sequence --- Cargo.lock | 10 ++++++++++ filter-parser/Cargo.toml | 1 + filter-parser/src/error.rs | 4 ++++ filter-parser/src/value.rs | 19 ++++++++++++++++++- 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 4aa7c7b67..26e0001b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1440,6 +1440,7 @@ dependencies = [ "insta", "nom", "nom_locate", + "unescaper", ] [[package]] @@ -4147,6 +4148,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" +[[package]] +name = "unescaper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96a44ae11e25afb520af4534fd7b0bd8cd613e35a78def813b8cf41631fa3c8" +dependencies = [ + "thiserror", +] + [[package]] name = "unicase" version = "2.6.0" diff --git a/filter-parser/Cargo.toml b/filter-parser/Cargo.toml index 58111ee08..d9b47f638 100644 --- a/filter-parser/Cargo.toml +++ b/filter-parser/Cargo.toml @@ -14,6 +14,7 @@ license.workspace = true [dependencies] nom = "7.1.3" nom_locate = "4.1.0" +unescaper = "0.1.2" [dev-dependencies] insta = "1.29.0" diff --git a/filter-parser/src/error.rs b/filter-parser/src/error.rs index b9d14a271..c71a3aaee 100644 --- a/filter-parser/src/error.rs +++ b/filter-parser/src/error.rs @@ -62,6 +62,7 @@ pub enum ErrorKind<'a> { MisusedGeoRadius, MisusedGeoBoundingBox, InvalidPrimary, + InvalidEscapedNumber, ExpectedEof, ExpectedValue(ExpectedValueKind), MalformedValue, @@ -147,6 +148,9 @@ impl<'a> Display for Error<'a> { let text = if input.trim().is_empty() { "but instead got nothing.".to_string() } else { format!("at `{}`.", escaped_input) }; writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` {}", text)? } + ErrorKind::InvalidEscapedNumber => { + writeln!(f, "Found an invalid escaped sequence number: `{}`.", escaped_input)? + } ErrorKind::ExpectedEof => { writeln!(f, "Found unexpected characters at the end of the filter: `{}`. You probably forgot an `OR` or an `AND` rule.", escaped_input)? } diff --git a/filter-parser/src/value.rs b/filter-parser/src/value.rs index 932aabca9..63d5ac384 100644 --- a/filter-parser/src/value.rs +++ b/filter-parser/src/value.rs @@ -171,7 +171,24 @@ pub fn parse_value(input: Span) -> IResult { }) })?; - Ok((input, value)) + match unescaper::unescape(value.value()) { + Ok(content) => { + if content.len() != value.value().len() { + Ok((input, Token::new(value.original_span(), Some(content)))) + } else { + Ok((input, value)) + } + } + Err(unescaper::Error::IncompleteStr(_)) => Err(nom::Err::Incomplete(nom::Needed::Unknown)), + Err(unescaper::Error::ParseIntError { .. }) => Err(nom::Err::Error(Error::new_from_kind( + value.original_span(), + ErrorKind::InvalidEscapedNumber, + ))), + Err(unescaper::Error::InvalidChar { .. }) => Err(nom::Err::Error(Error::new_from_kind( + value.original_span(), + ErrorKind::MalformedValue, + ))), + } } fn is_value_component(c: char) -> bool { From 66aa682e23cfb9538527038e3828d68932b63f32 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 7 Sep 2023 11:37:02 +0200 Subject: [PATCH 13/20] Register the swap indexe task in a spawn blocking to be sure to never block the main thread --- meilisearch/src/routes/swap_indexes.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/meilisearch/src/routes/swap_indexes.rs b/meilisearch/src/routes/swap_indexes.rs index c4e204c09..79e619705 100644 --- a/meilisearch/src/routes/swap_indexes.rs +++ b/meilisearch/src/routes/swap_indexes.rs @@ -60,8 +60,7 @@ pub async fn swap_indexes( } let task = KindWithContent::IndexSwap { swaps }; - - let task = index_scheduler.register(task)?; - let task: SummarizedTaskView = task.into(); + let task: SummarizedTaskView = + tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into(); Ok(HttpResponse::Accepted().json(task)) } From 651657c03e71992e8dbafd17e727b7d63fc6b102 Mon Sep 17 00:00:00 2001 From: curquiza Date: Thu, 7 Sep 2023 16:48:13 +0200 Subject: [PATCH 14/20] Fix git conflicts --- Cargo.lock | 28 ++++++++++++++-------------- Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e486648f3..03d491ff5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -469,7 +469,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "benchmarks" -version = "1.3.3" +version = "1.4.0" dependencies = [ "anyhow", "bytes", @@ -1199,7 +1199,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.3.3" +version = "1.4.0" dependencies = [ "anyhow", "big_s", @@ -1413,7 +1413,7 @@ dependencies = [ [[package]] name = "file-store" -version = "1.3.3" +version = "1.4.0" dependencies = [ "faux", "tempfile", @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.3.3" +version = "1.4.0" dependencies = [ "insta", "nom", @@ -1455,7 +1455,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.3.3" +version = "1.4.0" dependencies = [ "criterion", "serde_json", @@ -1573,7 +1573,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.3.3" +version = "1.4.0" dependencies = [ "arbitrary", "clap", @@ -1895,7 +1895,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "1.3.3" +version = "1.4.0" dependencies = [ "anyhow", "big_s", @@ -2082,7 +2082,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.3.3" +version = "1.4.0" dependencies = [ "criterion", "serde_json", @@ -2494,7 +2494,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.3.3" +version = "1.4.0" dependencies = [ "insta", "md5", @@ -2503,7 +2503,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.3.3" +version = "1.4.0" dependencies = [ "actix-cors", "actix-http", @@ -2592,7 +2592,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.3.3" +version = "1.4.0" dependencies = [ "base64 0.21.2", "enum-iterator", @@ -2611,7 +2611,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.3.3" +version = "1.4.0" dependencies = [ "actix-web", "anyhow", @@ -2665,7 +2665,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.3.3" +version = "1.4.0" dependencies = [ "big_s", "bimap", @@ -2995,7 +2995,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "permissive-json-pointer" -version = "1.3.3" +version = "1.4.0" dependencies = [ "big_s", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index afa79af5f..9c89aadfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ members = [ ] [workspace.package] -version = "1.3.3" +version = "1.4.0" authors = ["Quentin de Quelen ", "Clément Renault "] description = "Meilisearch HTTP server" homepage = "https://meilisearch.com" From 11df155598412240abdc851d37ad9bcd9b82b691 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Date: Mon, 4 Sep 2023 12:35:42 +0530 Subject: [PATCH 15/20] fix highlighting bug when searching for a phrase with cropping --- milli/src/search/new/matches/mod.rs | 33 +++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/milli/src/search/new/matches/mod.rs b/milli/src/search/new/matches/mod.rs index 72e155b3e..be4258e6a 100644 --- a/milli/src/search/new/matches/mod.rs +++ b/milli/src/search/new/matches/mod.rs @@ -418,19 +418,11 @@ impl<'t> Matcher<'t, '_> { } else { match &self.matches { Some((tokens, matches)) => { - // If the text has to be cropped, - // compute the best interval to crop around. - let matches = match format_options.crop { - Some(crop_size) if crop_size > 0 => { - self.find_best_match_interval(matches, crop_size) - } - _ => matches, - }; - // If the text has to be cropped, // crop around the best interval. let (byte_start, byte_end) = match format_options.crop { Some(crop_size) if crop_size > 0 => { + let matches = self.find_best_match_interval(matches, crop_size); self.crop_bounds(tokens, matches, crop_size) } _ => (0, self.text.len()), @@ -450,6 +442,11 @@ impl<'t> Matcher<'t, '_> { for m in matches { let token = &tokens[m.token_position]; + // skip matches out of the crop window. + if token.byte_start < byte_start || token.byte_end > byte_end { + continue; + } + if byte_index < token.byte_start { formatted.push(&self.text[byte_index..token.byte_start]); } @@ -800,6 +797,24 @@ mod tests { ); } + #[test] + fn format_highlight_crop_phrase_only() { + //! testing: https://github.com/meilisearch/meilisearch/issues/3975 + let temp_index = temp_index_with_documents(); + let rtxn = temp_index.read_txn().unwrap(); + let builder = MatcherBuilder::new_test(&rtxn, &temp_index, "\"the world\""); + + let format_options = FormatOptions { highlight: true, crop: Some(10) }; + + let text = "The groundbreaking invention had the power to split the world between those who embraced progress and those who resisted change!"; + let mut matcher = builder.build(text); + // should return 10 words with a marker at the start as well the end, and the highlighted matches. + insta::assert_snapshot!( + matcher.format(format_options), + @"…had the power to split the world between those who…" + ); + } + #[test] fn smaller_crop_size() { //! testing: https://github.com/meilisearch/specifications/pull/120#discussion_r836536295 From f2837aaec26afe9354ae0187d8992d51cf62a1bb Mon Sep 17 00:00:00 2001 From: Vivek Kumar Date: Tue, 5 Sep 2023 22:22:27 +0530 Subject: [PATCH 16/20] add another test case --- milli/src/search/new/matches/mod.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/milli/src/search/new/matches/mod.rs b/milli/src/search/new/matches/mod.rs index be4258e6a..ac6e49008 100644 --- a/milli/src/search/new/matches/mod.rs +++ b/milli/src/search/new/matches/mod.rs @@ -798,21 +798,29 @@ mod tests { } #[test] - fn format_highlight_crop_phrase_only() { + fn format_highlight_crop_phrase_query() { //! testing: https://github.com/meilisearch/meilisearch/issues/3975 let temp_index = temp_index_with_documents(); let rtxn = temp_index.read_txn().unwrap(); - let builder = MatcherBuilder::new_test(&rtxn, &temp_index, "\"the world\""); let format_options = FormatOptions { highlight: true, crop: Some(10) }; - let text = "The groundbreaking invention had the power to split the world between those who embraced progress and those who resisted change!"; + + let builder = MatcherBuilder::new_test(&rtxn, &temp_index, "\"the world\""); let mut matcher = builder.build(text); // should return 10 words with a marker at the start as well the end, and the highlighted matches. insta::assert_snapshot!( matcher.format(format_options), @"…had the power to split the world between those who…" ); + + let builder = MatcherBuilder::new_test(&rtxn, &temp_index, "those \"and those\""); + let mut matcher = builder.build(text); + // should highlight both "and" and "those". + insta::assert_snapshot!( + matcher.format(format_options), + @"…between those who embraced progress and those who resisted change…" + ); } #[test] From abfa7ded25ba3a94a772c552e9b11de034de4eaa Mon Sep 17 00:00:00 2001 From: Vivek Kumar Date: Wed, 6 Sep 2023 17:31:18 +0530 Subject: [PATCH 17/20] use a new temp index in the test --- milli/src/search/new/matches/mod.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/milli/src/search/new/matches/mod.rs b/milli/src/search/new/matches/mod.rs index ac6e49008..5d61de0f4 100644 --- a/milli/src/search/new/matches/mod.rs +++ b/milli/src/search/new/matches/mod.rs @@ -800,7 +800,12 @@ mod tests { #[test] fn format_highlight_crop_phrase_query() { //! testing: https://github.com/meilisearch/meilisearch/issues/3975 - let temp_index = temp_index_with_documents(); + let temp_index = TempIndex::new(); + temp_index + .add_documents(documents!([ + { "id": 1, "text": "The groundbreaking invention had the power to split the world between those who embraced progress and those who resisted change!" } + ])) + .unwrap(); let rtxn = temp_index.read_txn().unwrap(); let format_options = FormatOptions { highlight: true, crop: Some(10) }; @@ -816,10 +821,10 @@ mod tests { let builder = MatcherBuilder::new_test(&rtxn, &temp_index, "those \"and those\""); let mut matcher = builder.build(text); - // should highlight both "and" and "those". + // should highlight "those" and the phrase "and those". insta::assert_snapshot!( matcher.format(format_options), - @"…between those who embraced progress and those who resisted change…" + @"…world between those who embraced progress and those who resisted…" ); } From 9258e5b5bfe039fd1f5889f573f2770216d541a7 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 7 Sep 2023 18:22:42 +0200 Subject: [PATCH 18/20] Fix the stats of the documents deletion by filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue was that the operation « DocumentDeletionByFilter » was not declared as an index operation. That means the indexes stats were not reprocessed after the application of the operation. --- index-scheduler/src/batch.rs | 121 +++++++++--------- .../tests/documents/delete_documents.rs | 37 ++++++ 2 files changed, 98 insertions(+), 60 deletions(-) diff --git a/index-scheduler/src/batch.rs b/index-scheduler/src/batch.rs index fb865a98b..017a86fcf 100644 --- a/index-scheduler/src/batch.rs +++ b/index-scheduler/src/batch.rs @@ -67,10 +67,6 @@ pub(crate) enum Batch { op: IndexOperation, must_create_index: bool, }, - IndexDocumentDeletionByFilter { - index_uid: String, - task: Task, - }, IndexCreation { index_uid: String, primary_key: Option, @@ -114,6 +110,10 @@ pub(crate) enum IndexOperation { documents: Vec>, tasks: Vec, }, + IndexDocumentDeletionByFilter { + index_uid: String, + task: Task, + }, DocumentClear { index_uid: String, tasks: Vec, @@ -155,7 +155,6 @@ impl Batch { | Batch::TaskDeletion(task) | Batch::Dump(task) | Batch::IndexCreation { task, .. } - | Batch::IndexDocumentDeletionByFilter { task, .. } | Batch::IndexUpdate { task, .. } => vec![task.uid], Batch::SnapshotCreation(tasks) | Batch::IndexDeletion { tasks, .. } => { tasks.iter().map(|task| task.uid).collect() @@ -167,6 +166,7 @@ impl Batch { | IndexOperation::DocumentClear { tasks, .. } => { tasks.iter().map(|task| task.uid).collect() } + IndexOperation::IndexDocumentDeletionByFilter { task, .. } => vec![task.uid], IndexOperation::SettingsAndDocumentOperation { document_import_tasks: tasks, settings_tasks: other, @@ -194,8 +194,7 @@ impl Batch { IndexOperation { op, .. } => Some(op.index_uid()), IndexCreation { index_uid, .. } | IndexUpdate { index_uid, .. } - | IndexDeletion { index_uid, .. } - | IndexDocumentDeletionByFilter { index_uid, .. } => Some(index_uid), + | IndexDeletion { index_uid, .. } => Some(index_uid), } } } @@ -205,6 +204,7 @@ impl IndexOperation { match self { IndexOperation::DocumentOperation { index_uid, .. } | IndexOperation::DocumentDeletion { index_uid, .. } + | IndexOperation::IndexDocumentDeletionByFilter { index_uid, .. } | IndexOperation::DocumentClear { index_uid, .. } | IndexOperation::Settings { index_uid, .. } | IndexOperation::DocumentClearAndSetting { index_uid, .. } @@ -239,9 +239,12 @@ impl IndexScheduler { let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?; match &task.kind { KindWithContent::DocumentDeletionByFilter { index_uid, .. } => { - Ok(Some(Batch::IndexDocumentDeletionByFilter { - index_uid: index_uid.clone(), - task, + Ok(Some(Batch::IndexOperation { + op: IndexOperation::IndexDocumentDeletionByFilter { + index_uid: index_uid.clone(), + task, + }, + must_create_index: false, })) } _ => unreachable!(), @@ -896,51 +899,6 @@ impl IndexScheduler { Ok(tasks) } - Batch::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => { - let (index_uid, filter) = - if let KindWithContent::DocumentDeletionByFilter { index_uid, filter_expr } = - &task.kind - { - (index_uid, filter_expr) - } else { - unreachable!() - }; - let index = { - let rtxn = self.env.read_txn()?; - self.index_mapper.index(&rtxn, index_uid)? - }; - let deleted_documents = delete_document_by_filter(filter, index); - let original_filter = if let Some(Details::DocumentDeletionByFilter { - original_filter, - deleted_documents: _, - }) = task.details - { - original_filter - } else { - // In the case of a `documentDeleteByFilter` the details MUST be set - unreachable!(); - }; - - match deleted_documents { - Ok(deleted_documents) => { - task.status = Status::Succeeded; - task.details = Some(Details::DocumentDeletionByFilter { - original_filter, - deleted_documents: Some(deleted_documents), - }); - } - Err(e) => { - task.status = Status::Failed; - task.details = Some(Details::DocumentDeletionByFilter { - original_filter, - deleted_documents: Some(0), - }); - task.error = Some(e.into()); - } - } - - Ok(vec![task]) - } Batch::IndexCreation { index_uid, primary_key, task } => { let wtxn = self.env.write_txn()?; if self.index_mapper.exists(&wtxn, &index_uid)? { @@ -1299,6 +1257,47 @@ impl IndexScheduler { Ok(tasks) } + IndexOperation::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => { + let filter = + if let KindWithContent::DocumentDeletionByFilter { filter_expr, .. } = + &task.kind + { + filter_expr + } else { + unreachable!() + }; + let deleted_documents = delete_document_by_filter(index_wtxn, filter, index); + let original_filter = if let Some(Details::DocumentDeletionByFilter { + original_filter, + deleted_documents: _, + }) = task.details + { + original_filter + } else { + // In the case of a `documentDeleteByFilter` the details MUST be set + unreachable!(); + }; + + match deleted_documents { + Ok(deleted_documents) => { + task.status = Status::Succeeded; + task.details = Some(Details::DocumentDeletionByFilter { + original_filter, + deleted_documents: Some(deleted_documents), + }); + } + Err(e) => { + task.status = Status::Failed; + task.details = Some(Details::DocumentDeletionByFilter { + original_filter, + deleted_documents: Some(0), + }); + task.error = Some(e.into()); + } + } + + Ok(vec![task]) + } IndexOperation::Settings { index_uid: _, settings, mut tasks } => { let indexer_config = self.index_mapper.indexer_config(); let mut builder = milli::update::Settings::new(index_wtxn, index, indexer_config); @@ -1498,22 +1497,24 @@ impl IndexScheduler { } } -fn delete_document_by_filter(filter: &serde_json::Value, index: Index) -> Result { +fn delete_document_by_filter<'a>( + wtxn: &mut RwTxn<'a, '_>, + filter: &serde_json::Value, + index: &'a Index, +) -> Result { let filter = Filter::from_json(filter)?; Ok(if let Some(filter) = filter { - let mut wtxn = index.write_txn()?; - let candidates = filter.evaluate(&wtxn, &index).map_err(|err| match err { milli::Error::UserError(milli::UserError::InvalidFilter(_)) => { Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter) } e => e.into(), })?; - let mut delete_operation = DeleteDocuments::new(&mut wtxn, &index)?; + let mut delete_operation = DeleteDocuments::new(wtxn, index)?; delete_operation.delete_documents(&candidates); + let deleted_documents = delete_operation.execute().map(|result| result.deleted_documents)?; - wtxn.commit()?; deleted_documents } else { 0 diff --git a/meilisearch/tests/documents/delete_documents.rs b/meilisearch/tests/documents/delete_documents.rs index 8f6ae1985..c2704dd1a 100644 --- a/meilisearch/tests/documents/delete_documents.rs +++ b/meilisearch/tests/documents/delete_documents.rs @@ -154,6 +154,19 @@ async fn delete_document_by_filter() { ) .await; index.wait_task(1).await; + + let (stats, _) = index.stats().await; + snapshot!(json_string!(stats), @r###" + { + "numberOfDocuments": 4, + "isIndexing": false, + "fieldDistribution": { + "color": 3, + "id": 4 + } + } + "###); + let (response, code) = index.delete_document_by_filter(json!({ "filter": "color = blue"})).await; snapshot!(code, @"202 Accepted"); @@ -188,6 +201,18 @@ async fn delete_document_by_filter() { } "###); + let (stats, _) = index.stats().await; + snapshot!(json_string!(stats), @r###" + { + "numberOfDocuments": 2, + "isIndexing": false, + "fieldDistribution": { + "color": 1, + "id": 2 + } + } + "###); + let (documents, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; snapshot!(code, @"200 OK"); snapshot!(json_string!(documents), @r###" @@ -241,6 +266,18 @@ async fn delete_document_by_filter() { } "###); + let (stats, _) = index.stats().await; + snapshot!(json_string!(stats), @r###" + { + "numberOfDocuments": 1, + "isIndexing": false, + "fieldDistribution": { + "color": 1, + "id": 1 + } + } + "###); + let (documents, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; snapshot!(code, @"200 OK"); snapshot!(json_string!(documents), @r###" From 393be40179cc3981ab3c69745162853586d13778 Mon Sep 17 00:00:00 2001 From: dogukanakkaya Date: Mon, 14 Aug 2023 01:22:14 +0300 Subject: [PATCH 19/20] Refactor empty arrays/objects should return empty instead of null --- permissive-json-pointer/src/lib.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/permissive-json-pointer/src/lib.rs b/permissive-json-pointer/src/lib.rs index 7e5b3371c..c771f3321 100644 --- a/permissive-json-pointer/src/lib.rs +++ b/permissive-json-pointer/src/lib.rs @@ -186,12 +186,16 @@ fn create_value(value: &Document, mut selectors: HashSet<&str>) -> Document { let array = create_array(array, &sub_selectors); if !array.is_empty() { new_value.insert(key.to_string(), array.into()); + } else { + new_value.insert(key.to_string(), Value::Array(vec![])); } } Value::Object(object) => { let object = create_value(object, sub_selectors); if !object.is_empty() { new_value.insert(key.to_string(), object.into()); + } else { + new_value.insert(key.to_string(), Value::Object(Map::new())); } } _ => (), @@ -211,6 +215,8 @@ fn create_array(array: &[Value], selectors: &HashSet<&str>) -> Vec { let array = create_array(array, selectors); if !array.is_empty() { res.push(array.into()); + } else { + res.push(Value::Array(vec![])); } } Value::Object(object) => { @@ -637,6 +643,24 @@ mod tests { ); } + #[test] + fn empty_array_object_return_empty() { + let value: Value = json!({ + "array": [], + "object": {}, + }); + let value: &Document = value.as_object().unwrap(); + + let res: Value = select_values(value, vec!["array.name", "object.name"]).into(); + assert_eq!( + res, + json!({ + "array": [], + "object": {}, + }) + ); + } + #[test] fn all_conflict_variation() { let value: Value = json!({ From 34fac115d5e3f66364114a9ea4ad033584c3793a Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Sep 2023 17:15:57 +0200 Subject: [PATCH 20/20] fix clippy --- index-scheduler/src/batch.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/index-scheduler/src/batch.rs b/index-scheduler/src/batch.rs index 017a86fcf..ccdcbcbb6 100644 --- a/index-scheduler/src/batch.rs +++ b/index-scheduler/src/batch.rs @@ -1504,7 +1504,7 @@ fn delete_document_by_filter<'a>( ) -> Result { let filter = Filter::from_json(filter)?; Ok(if let Some(filter) = filter { - let candidates = filter.evaluate(&wtxn, &index).map_err(|err| match err { + let candidates = filter.evaluate(wtxn, index).map_err(|err| match err { milli::Error::UserError(milli::UserError::InvalidFilter(_)) => { Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter) } @@ -1512,10 +1512,7 @@ fn delete_document_by_filter<'a>( })?; let mut delete_operation = DeleteDocuments::new(wtxn, index)?; delete_operation.delete_documents(&candidates); - - let deleted_documents = - delete_operation.execute().map(|result| result.deleted_documents)?; - deleted_documents + delete_operation.execute().map(|result| result.deleted_documents)? } else { 0 })