From 43df4a56c4014f952f76bd975a296ef885078345 Mon Sep 17 00:00:00 2001 From: Balaji Sivaraman Date: Sat, 11 Jul 2020 15:46:29 +0530 Subject: [PATCH 1/2] feat(search): update nbHits count with filtered docs for core flow --- meilisearch-core/src/bucket_sort.rs | 26 +++++++++++++--- meilisearch-http/tests/search.rs | 48 +++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/meilisearch-core/src/bucket_sort.rs b/meilisearch-core/src/bucket_sort.rs index f0fc71bd0..57e50b87f 100644 --- a/meilisearch-core/src/bucket_sort.rs +++ b/meilisearch-core/src/bucket_sort.rs @@ -212,6 +212,7 @@ where FD: Fn(DocumentId) -> Option, { let mut result = SortResult::default(); + let mut filtered_count = 0; let words_set = index.main.words_fst(reader)?; let stop_words = index.main.stop_words_fst(reader)?; @@ -322,19 +323,36 @@ where let filter_accepted = match &filter { Some(filter) => { let entry = filter_map.entry(document.id); - *entry.or_insert_with(|| (filter)(document.id)) + *entry.or_insert_with(|| { + let accepted = (filter)(document.id); + // we only want to count it out the first time we see it + if !accepted { + filtered_count += 1; + } + accepted + }) } None => true, }; if filter_accepted { let entry = key_cache.entry(document.id); - let key = entry.or_insert_with(|| (distinct)(document.id).map(Rc::new)); + let mut seen = true; + let key = entry.or_insert_with(|| { + seen = false; + (distinct)(document.id).map(Rc::new) + }); - match key.clone() { + let distinct = match key.clone() { Some(key) => buf_distinct.register(key), None => buf_distinct.register_without_key(), }; + + // we only want to count the document if it is the first time we see it and + // if it wasn't accepted by distinct + if !seen && !distinct { + filtered_count += 1; + } } // the requested range end is reached: stop computing distinct @@ -396,7 +414,7 @@ where } } result.documents = documents; - result.nb_hits = docids.len(); + result.nb_hits = docids.len() - filtered_count; Ok(result) } diff --git a/meilisearch-http/tests/search.rs b/meilisearch-http/tests/search.rs index 170d9ffaf..06cca487d 100644 --- a/meilisearch-http/tests/search.rs +++ b/meilisearch-http/tests/search.rs @@ -1829,3 +1829,51 @@ async fn update_documents_with_facet_distribution() { let (response2, _) = server.search_post(search).await; assert_json_eq!(expected_facet_distribution, response2["facetsDistribution"].clone()); } + +#[actix_rt::test] +async fn test_filter_nb_hits_search_normal() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + + server.create_index(body).await; + let documents = json!([ + { + "id": 1, + "content": "a", + "color": "green", + "size": 1, + }, + { + "id": 2, + "content": "a", + "color": "green", + "size": 2, + }, + { + "id": 3, + "content": "a", + "color": "blue", + "size": 3, + }, + ]); + + server.add_or_update_multiple_documents(documents).await; + let (response, _) = server.search_post(json!({"q": "a"})).await; + assert_eq!(response["nbHits"], 3); + + let (response, _) = server.search_post(json!({"q": "a", "filters": "size = 1"})).await; + assert_eq!(response["nbHits"], 1); + + server.update_distinct_attribute(json!("color")).await; + + let (response, _) = server.search_post(json!({"q": "a"})).await; + assert_eq!(response["nbHits"], 2); + + let (response, _) = server.search_post(json!({"q": "a", "filters": "size < 3"})).await; + println!("result: {}", response); + assert_eq!(response["nbHits"], 1); +} From 75e22fc7f5e5e0468dbbce681b9c461c97e9dc44 Mon Sep 17 00:00:00 2001 From: Balaji Sivaraman Date: Mon, 20 Jul 2020 20:33:46 +0530 Subject: [PATCH 2/2] feat(search): update nbHits count with filtered docs for placeholder search --- meilisearch-core/src/query_builder.rs | 17 ++++++-- meilisearch-http/tests/placeholder_search.rs | 45 ++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/meilisearch-core/src/query_builder.rs b/meilisearch-core/src/query_builder.rs index 40cabc84c..6f4485342 100644 --- a/meilisearch-core/src/query_builder.rs +++ b/meilisearch-core/src/query_builder.rs @@ -225,10 +225,17 @@ impl<'c, 'f, 'd, 'i> QueryBuilder<'c, 'f, 'd, 'i> { fn sort_result_from_docids(&self, docids: &[DocumentId], range: Range) -> SortResult { let mut sort_result = SortResult::default(); + let mut filtered_count = 0; let mut result = match self.filter { Some(ref filter) => docids .iter() - .filter(|item| (filter)(**item)) + .filter(|item| { + let accepted = (filter)(**item); + if !accepted { + filtered_count += 1; + } + accepted + }) .skip(range.start) .take(range.end - range.start) .map(|&id| Document::from_highlights(id, &[])) @@ -248,15 +255,19 @@ impl<'c, 'f, 'd, 'i> QueryBuilder<'c, 'f, 'd, 'i> { result.retain(|doc| { let id = doc.id; let key = (distinct)(id); - match key { + let distinct_accepted = match key { Some(key) => distinct_map.register(key), None => distinct_map.register_without_key(), + }; + if !distinct_accepted { + filtered_count += 1; } + distinct_accepted }); } sort_result.documents = result; - sort_result.nb_hits = docids.len(); + sort_result.nb_hits = docids.len() - filtered_count; sort_result } diff --git a/meilisearch-http/tests/placeholder_search.rs b/meilisearch-http/tests/placeholder_search.rs index dde85b470..a586762f3 100644 --- a/meilisearch-http/tests/placeholder_search.rs +++ b/meilisearch-http/tests/placeholder_search.rs @@ -588,3 +588,48 @@ async fn placeholder_search_with_empty_query() { assert_eq!(response["hits"].as_array().unwrap().len(), 3); }); } + +#[actix_rt::test] +async fn test_filter_nb_hits_search_placeholder() { + let mut server = common::Server::with_uid("test"); + + let body = json!({ + "uid": "test", + "primaryKey": "id", + }); + + server.create_index(body).await; + let documents = json!([ + { + "id": 1, + "content": "a", + "color": "green", + "size": 1, + }, + { + "id": 2, + "content": "a", + "color": "green", + "size": 2, + }, + { + "id": 3, + "content": "a", + "color": "blue", + "size": 3, + }, + ]); + + server.add_or_update_multiple_documents(documents).await; + let (response, _) = server.search_post(json!({})).await; + assert_eq!(response["nbHits"], 3); + + server.update_distinct_attribute(json!("color")).await; + + let (response, _) = server.search_post(json!({})).await; + assert_eq!(response["nbHits"], 2); + + let (response, _) = server.search_post(json!({"filters": "size < 3"})).await; + println!("result: {}", response); + assert_eq!(response["nbHits"], 1); +}