From 6e9d7f94d40361077c40bd9653bb7b9e91680b16 Mon Sep 17 00:00:00 2001 From: mposmta Date: Wed, 25 Mar 2020 12:11:37 +0100 Subject: [PATCH 1/4] adds exhaustive number hits to search result --- meilisearch-http/src/helpers/meilisearch.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meilisearch-http/src/helpers/meilisearch.rs b/meilisearch-http/src/helpers/meilisearch.rs index 377bb1ae9..9aa5174ff 100644 --- a/meilisearch-http/src/helpers/meilisearch.rs +++ b/meilisearch-http/src/helpers/meilisearch.rs @@ -282,6 +282,8 @@ impl<'a> SearchBuilder<'a> { hits, offset: self.offset, limit: self.limit, + nb_hits: 0, + exhaustive_nb_hits: false, processing_time_ms: time_ms, query: self.query.to_string(), }; @@ -364,6 +366,8 @@ pub struct SearchResult { pub hits: Vec, pub offset: usize, pub limit: usize, + pub nb_hits: usize, + pub exhaustive_nb_hits: bool, pub processing_time_ms: usize, pub query: String, } From fd65cf9dcb842ebd7eb32bddf9966db838f3809e Mon Sep 17 00:00:00 2001 From: mposmta Date: Wed, 25 Mar 2020 12:44:38 +0100 Subject: [PATCH 2/4] populates exhaustive number of hits --- meilisearch-core/src/bucket_sort.rs | 12 ++++++------ meilisearch-core/src/query_builder.rs | 2 +- meilisearch-http/src/helpers/meilisearch.rs | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/meilisearch-core/src/bucket_sort.rs b/meilisearch-core/src/bucket_sort.rs index 4c34f1abd..23e7149a2 100644 --- a/meilisearch-core/src/bucket_sort.rs +++ b/meilisearch-core/src/bucket_sort.rs @@ -37,7 +37,7 @@ pub fn bucket_sort<'c, FI>( synonyms_store: store::Synonyms, prefix_documents_cache_store: store::PrefixDocumentsCache, prefix_postings_lists_cache_store: store::PrefixPostingsListsCache, -) -> MResult> +) -> MResult<(Vec, usize)> where FI: Fn(DocumentId) -> bool, { @@ -66,7 +66,7 @@ where let words_set = match unsafe { main_store.static_words_fst(reader)? } { Some(words) => words, - None => return Ok(Vec::new()), + None => return Ok((Vec::new(), 0)), }; let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default(); @@ -172,7 +172,7 @@ where debug!("bucket sort took {:.02?}", before_bucket_sort.elapsed()); - Ok(documents) + Ok((documents, docids.len())) } pub fn bucket_sort_with_distinct<'c, FI, FD>( @@ -190,14 +190,14 @@ pub fn bucket_sort_with_distinct<'c, FI, FD>( synonyms_store: store::Synonyms, _prefix_documents_cache_store: store::PrefixDocumentsCache, prefix_postings_lists_cache_store: store::PrefixPostingsListsCache, -) -> MResult> +) -> MResult<(Vec, usize)> where FI: Fn(DocumentId) -> bool, FD: Fn(DocumentId) -> Option, { let words_set = match unsafe { main_store.static_words_fst(reader)? } { Some(words) => words, - None => return Ok(Vec::new()), + None => return Ok((Vec::new(), 0)), }; let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default(); @@ -363,7 +363,7 @@ where } } - Ok(documents) + Ok((documents, docids.len())) } fn cleanup_bare_matches<'tag, 'txn>( diff --git a/meilisearch-core/src/query_builder.rs b/meilisearch-core/src/query_builder.rs index 684e3b1a8..d73f3b4dd 100644 --- a/meilisearch-core/src/query_builder.rs +++ b/meilisearch-core/src/query_builder.rs @@ -92,7 +92,7 @@ impl<'c, 'f, 'd> QueryBuilder<'c, 'f, 'd> { reader: &heed::RoTxn, query: &str, range: Range, - ) -> MResult> { + ) -> MResult<(Vec, usize)> { match self.distinct { Some((distinct, distinct_size)) => bucket_sort_with_distinct( reader, diff --git a/meilisearch-http/src/helpers/meilisearch.rs b/meilisearch-http/src/helpers/meilisearch.rs index 9aa5174ff..64140b3aa 100644 --- a/meilisearch-http/src/helpers/meilisearch.rs +++ b/meilisearch-http/src/helpers/meilisearch.rs @@ -223,12 +223,12 @@ impl<'a> SearchBuilder<'a> { } let start = Instant::now(); - let docs = - query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit)); + let result = query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit)); let time_ms = start.elapsed().as_millis() as usize; + let (docs, nb_hits) = result.map_err(|e| Error::SearchDocuments(e.to_string()))?; let mut hits = Vec::with_capacity(self.limit); - for doc in docs.map_err(|e| Error::SearchDocuments(e.to_string()))? { + for doc in docs { // retrieve the content of document in kv store let mut fields: Option> = None; if let Some(attributes_to_retrieve) = &self.attributes_to_retrieve { @@ -282,7 +282,7 @@ impl<'a> SearchBuilder<'a> { hits, offset: self.offset, limit: self.limit, - nb_hits: 0, + nb_hits, exhaustive_nb_hits: false, processing_time_ms: time_ms, query: self.query.to_string(), From fa9b7dd29f7d82de3ad405b23ad7d043a8295041 Mon Sep 17 00:00:00 2001 From: mposmta Date: Wed, 25 Mar 2020 13:59:15 +0100 Subject: [PATCH 3/4] removes useless deserializer for SearchResult --- meilisearch-http/src/helpers/meilisearch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/helpers/meilisearch.rs b/meilisearch-http/src/helpers/meilisearch.rs index 64140b3aa..88946c7ac 100644 --- a/meilisearch-http/src/helpers/meilisearch.rs +++ b/meilisearch-http/src/helpers/meilisearch.rs @@ -360,7 +360,7 @@ pub struct SearchHit { pub matches_info: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct SearchResult { pub hits: Vec, From a6dcd7a42182a2fd478dc3739963e7711946e3e8 Mon Sep 17 00:00:00 2001 From: mposmta Date: Wed, 25 Mar 2020 14:00:29 +0100 Subject: [PATCH 4/4] fixes tests fixes tests impacted by sifnature change of query --- meilisearch-core/examples/from_file.rs | 2 +- meilisearch-core/src/database.rs | 4 +- meilisearch-core/src/query_builder.rs | 56 ++++++++++----------- meilisearch-http/src/helpers/meilisearch.rs | 2 +- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/meilisearch-core/examples/from_file.rs b/meilisearch-core/examples/from_file.rs index 3762c99f0..2a30d1ce4 100644 --- a/meilisearch-core/examples/from_file.rs +++ b/meilisearch-core/examples/from_file.rs @@ -371,7 +371,7 @@ fn search_command(command: SearchCommand, database: Database) -> Result<(), Box< }); } - let documents = builder.query(ref_reader, &query, 0..command.number_results)?; + let (documents, _nb_hits) = builder.query(ref_reader, &query, 0..command.number_results)?; let mut retrieve_duration = Duration::default(); diff --git a/meilisearch-core/src/database.rs b/meilisearch-core/src/database.rs index 09daf4864..87fc24c11 100644 --- a/meilisearch-core/src/database.rs +++ b/meilisearch-core/src/database.rs @@ -661,7 +661,7 @@ mod tests { // even try to search for a document let reader = db.main_read_txn().unwrap(); - let results = index.query_builder().query(&reader, "21 ", 0..20).unwrap(); + let (results, _nb_hits) = index.query_builder().query(&reader, "21 ", 0..20).unwrap(); assert_matches!(results.len(), 1); reader.abort(); @@ -1059,7 +1059,7 @@ mod tests { let builder = index.query_builder_with_criteria(criteria); - let results = builder.query(&reader, "Kevin", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "Kevin", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!( diff --git a/meilisearch-core/src/query_builder.rs b/meilisearch-core/src/query_builder.rs index d73f3b4dd..68ed97537 100644 --- a/meilisearch-core/src/query_builder.rs +++ b/meilisearch-core/src/query_builder.rs @@ -331,7 +331,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "iphone from apple", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "iphone from apple", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -354,7 +354,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "hello", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "hello", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -365,7 +365,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "bonjour", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "bonjour", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -469,7 +469,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "hello", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "hello", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -490,7 +490,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "bonjour", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "bonjour", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -511,7 +511,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "salut", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "salut", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -557,7 +557,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "NY subway", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "NY subway", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { @@ -579,7 +579,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "NYC subway", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "NYC subway", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { @@ -621,7 +621,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "NY", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "NY", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(2), matches, .. }) => { @@ -645,7 +645,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "new york", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "new york", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -679,7 +679,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "NY subway", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "NY subway", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -696,7 +696,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "new york subway", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "new york subway", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { @@ -744,7 +744,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "NY subway", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "NY subway", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { @@ -766,7 +766,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "NYC subway", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "NYC subway", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { @@ -819,7 +819,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "NY subway broken", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "NY subway broken", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -835,7 +835,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "NYC subway", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "NYC subway", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { @@ -891,7 +891,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder + let (results, _nb_hits) = builder .query(&reader, "new york underground train broken", 0..20) .unwrap(); let mut iter = results.into_iter(); @@ -921,7 +921,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder + let (results, _nb_hits) = builder .query(&reader, "new york city underground train broken", 0..20) .unwrap(); let mut iter = results.into_iter(); @@ -965,7 +965,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "new york big ", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "new york big ", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -999,7 +999,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "NY subway ", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "NY subway ", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -1049,7 +1049,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder + let (results, _nb_hits) = builder .query(&reader, "new york city long subway cool ", 0..20) .unwrap(); let mut iter = results.into_iter(); @@ -1082,7 +1082,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "telephone", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "telephone", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -1099,7 +1099,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "téléphone", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "téléphone", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -1116,7 +1116,7 @@ mod tests { assert_matches!(iter.next(), None); let builder = store.query_builder(); - let results = builder.query(&reader, "télephone", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "télephone", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(1), matches, .. }) => { @@ -1143,7 +1143,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "i phone case", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "i phone case", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -1172,7 +1172,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "searchengine", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "searchengine", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -1212,7 +1212,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "searchengine", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "searchengine", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { @@ -1244,7 +1244,7 @@ mod tests { let reader = db.main_read_txn().unwrap(); let builder = store.query_builder(); - let results = builder.query(&reader, "searchengine", 0..20).unwrap(); + let (results, _nb_hits) = builder.query(&reader, "searchengine", 0..20).unwrap(); let mut iter = results.into_iter(); assert_matches!(iter.next(), Some(Document { id: DocumentId(0), matches, .. }) => { diff --git a/meilisearch-http/src/helpers/meilisearch.rs b/meilisearch-http/src/helpers/meilisearch.rs index 88946c7ac..31510ec49 100644 --- a/meilisearch-http/src/helpers/meilisearch.rs +++ b/meilisearch-http/src/helpers/meilisearch.rs @@ -224,8 +224,8 @@ impl<'a> SearchBuilder<'a> { let start = Instant::now(); let result = query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit)); - let time_ms = start.elapsed().as_millis() as usize; let (docs, nb_hits) = result.map_err(|e| Error::SearchDocuments(e.to_string()))?; + let time_ms = start.elapsed().as_millis() as usize; let mut hits = Vec::with_capacity(self.limit); for doc in docs {