add sort result struct

This commit is contained in:
mpostma 2020-05-07 19:25:18 +02:00
parent a88f6c3241
commit effbb7f7f1
7 changed files with 194 additions and 69 deletions

92
Cargo.lock generated
View File

@ -522,9 +522,9 @@ dependencies = [
[[package]] [[package]]
name = "bstr" name = "bstr"
version = "0.2.12" version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2889e6d50f394968c8bf4240dc3f2a7eb4680844d27308f798229ac9d4725f41" checksum = "31accafdb70df7871592c058eca3985b71104e15ac32f64706022c58867da931"
dependencies = [ dependencies = [
"lazy_static", "lazy_static",
"memchr", "memchr",
@ -616,9 +616,9 @@ checksum = "5b89647f09b9f4c838cb622799b2843e4e13bff64661dab9a0362bb92985addd"
[[package]] [[package]]
name = "clap" name = "clap"
version = "2.33.0" version = "2.33.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129"
dependencies = [ dependencies = [
"ansi_term", "ansi_term",
"atty", "atty",
@ -1008,9 +1008,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c329ae8753502fb44ae4fc2b622fa2a94652c41e795143765ba0927f92ab780" checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -1023,9 +1023,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -1033,15 +1033,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399"
[[package]] [[package]]
name = "futures-executor" name = "futures-executor"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f674f3e1bcb15b37284a90cedf55afdba482ab061c407a9c0ebbd0f3109741ba" checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-task", "futures-task",
@ -1050,15 +1050,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789"
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39"
dependencies = [ dependencies = [
"proc-macro-hack", "proc-macro-hack",
"proc-macro2", "proc-macro2",
@ -1068,21 +1068,24 @@ dependencies = [
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc"
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626"
dependencies = [
"once_cell",
]
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -1091,6 +1094,7 @@ dependencies = [
"futures-sink", "futures-sink",
"futures-task", "futures-task",
"memchr", "memchr",
"pin-project",
"pin-utils", "pin-utils",
"proc-macro-hack", "proc-macro-hack",
"proc-macro-nested", "proc-macro-nested",
@ -1398,9 +1402,9 @@ dependencies = [
[[package]] [[package]]
name = "intervaltree" name = "intervaltree"
version = "0.2.5" version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8254add2ea664734c9d001f8151cc3d7696b135f7e40e5a2efa814a662cb3a44" checksum = "566d5aa3b5cc5c5809cc1a9c9588d917a634248bfc58f7ea14e354e71595a32c"
dependencies = [ dependencies = [
"smallvec", "smallvec",
] ]
@ -1842,9 +1846,9 @@ dependencies = [
[[package]] [[package]]
name = "ntapi" name = "ntapi"
version = "0.3.3" version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26e041cd983acbc087e30fcba770380cfa352d0e392e175b2344ebaf7ea0602" checksum = "7a31937dea023539c72ddae0e3571deadc1414b300483fa7aaec176168cfa9d2"
dependencies = [ dependencies = [
"winapi 0.3.8", "winapi 0.3.8",
] ]
@ -1918,9 +1922,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
[[package]] [[package]]
name = "openssl-sys" name = "openssl-sys"
version = "0.9.55" version = "0.9.56"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7717097d810a0f2e2323f9e5d11e71608355e24828410b55b9d4f18aa5f9a5d8" checksum = "f02309a7f127000ed50594f0b50ecc69e7c654e16d41b4e8156d1b3df8e0b52e"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"cc", "cc",
@ -2038,18 +2042,18 @@ dependencies = [
[[package]] [[package]]
name = "pin-project" name = "pin-project"
version = "0.4.13" version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82c3bfbfb5bb42f99498c7234bbd768c220eb0cea6818259d0d18a1aa3d2595d" checksum = "81d480cb4e89522ccda96d0eed9af94180b7a5f93fb28f66e1fd7d68431663d1"
dependencies = [ dependencies = [
"pin-project-internal", "pin-project-internal",
] ]
[[package]] [[package]]
name = "pin-project-internal" name = "pin-project-internal"
version = "0.4.13" version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccbf6449dcfb18562c015526b085b8df1aa3cdab180af8ec2ebd300a3bd28f63" checksum = "a82996f11efccb19b685b14b5df818de31c1edcee3daa256ab5775dd98e72feb"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2466,9 +2470,9 @@ dependencies = [
[[package]] [[package]]
name = "schannel" name = "schannel"
version = "0.1.18" version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "039c25b130bd8c1321ee2d7de7fde2659fa9c2744e4bb29711cfc852ea53cd19" checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
dependencies = [ dependencies = [
"lazy_static", "lazy_static",
"winapi 0.3.8", "winapi 0.3.8",
@ -2498,9 +2502,9 @@ checksum = "cbb21fe0588557792176c89bc7b943027b14f346d03c6be6a199c2860277d93a"
[[package]] [[package]]
name = "security-framework" name = "security-framework"
version = "0.4.3" version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f331b9025654145cd425b9ded0caf8f5ae0df80d418b326e2dc1c3dc5eb0620" checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"core-foundation", "core-foundation",
@ -2575,18 +2579,18 @@ dependencies = [
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.106" version = "1.0.110"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c"
dependencies = [ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.106" version = "1.0.110"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2595,9 +2599,9 @@ dependencies = [
[[package]] [[package]]
name = "serde_json" name = "serde_json"
version = "1.0.52" version = "1.0.53"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7894c8ed05b7a3a279aeb79025fdec1d3158080b75b98a08faf2806bb799edd" checksum = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"itoa", "itoa",
@ -2853,9 +2857,9 @@ dependencies = [
[[package]] [[package]]
name = "threadpool" name = "threadpool"
version = "1.8.0" version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8dae184447c15d5a6916d973c642aec485105a13cd238192a6927ae3e077d66" checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
dependencies = [ dependencies = [
"num_cpus", "num_cpus",
] ]

View File

@ -11,12 +11,13 @@ use std::fmt;
use compact_arena::{SmallArena, Idx32, mk_arena}; use compact_arena::{SmallArena, Idx32, mk_arena};
use log::debug; use log::debug;
use meilisearch_types::DocIndex; use meilisearch_types::DocIndex;
use sdset::{Set, SetBuf, exponential_search, SetOperation}; use sdset::{Set, SetBuf, exponential_search, SetOperation, Counter, duo::OpBuilder};
use slice_group_by::{GroupBy, GroupByMut}; use slice_group_by::{GroupBy, GroupByMut};
use crate::error::Error; use crate::error::Error;
use crate::criterion::{Criteria, Context, ContextMut}; use crate::criterion::{Criteria, Context, ContextMut};
use crate::distinct_map::{BufferedDistinctMap, DistinctMap}; use crate::distinct_map::{BufferedDistinctMap, DistinctMap};
use crate::facets::FacetKey;
use crate::raw_document::RawDocument; use crate::raw_document::RawDocument;
use crate::{database::MainT, reordered_attrs::ReorderedAttrs}; use crate::{database::MainT, reordered_attrs::ReorderedAttrs};
use crate::{store, Document, DocumentId, MResult}; use crate::{store, Document, DocumentId, MResult};
@ -24,11 +25,20 @@ use crate::query_tree::{create_query_tree, traverse_query_tree};
use crate::query_tree::{Operation, QueryResult, QueryKind, QueryId, PostingsKey}; use crate::query_tree::{Operation, QueryResult, QueryKind, QueryId, PostingsKey};
use crate::query_tree::Context as QTContext; use crate::query_tree::Context as QTContext;
#[derive(Debug, Default)]
pub struct SortResult {
pub documents: Vec<Document>,
pub nb_hits: usize,
pub is_exhaustive: bool,
pub facets: Option<HashMap<FacetKey, usize>>,
}
pub fn bucket_sort<'c, FI>( pub fn bucket_sort<'c, FI>(
reader: &heed::RoTxn<MainT>, reader: &heed::RoTxn<MainT>,
query: &str, query: &str,
range: Range<usize>, range: Range<usize>,
facets_docids: Option<SetBuf<DocumentId>>, facets_docids: Option<SetBuf<DocumentId>>,
facet_count_docids: Option<HashMap<FacetKey, Cow<Set<DocumentId>>>>,
filter: Option<FI>, filter: Option<FI>,
criteria: Criteria<'c>, criteria: Criteria<'c>,
searchable_attrs: Option<ReorderedAttrs>, searchable_attrs: Option<ReorderedAttrs>,
@ -38,7 +48,7 @@ pub fn bucket_sort<'c, FI>(
synonyms_store: store::Synonyms, synonyms_store: store::Synonyms,
prefix_documents_cache_store: store::PrefixDocumentsCache, prefix_documents_cache_store: store::PrefixDocumentsCache,
prefix_postings_lists_cache_store: store::PrefixPostingsListsCache, prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
) -> MResult<(Vec<Document>, usize)> ) -> MResult<SortResult>
where where
FI: Fn(DocumentId) -> bool, FI: Fn(DocumentId) -> bool,
{ {
@ -52,6 +62,7 @@ where
query, query,
range, range,
facets_docids, facets_docids,
facet_count_docids,
filter, filter,
distinct, distinct,
distinct_size, distinct_size,
@ -66,9 +77,11 @@ where
); );
} }
let mut result = SortResult::default();
let words_set = match unsafe { main_store.static_words_fst(reader)? } { let words_set = match unsafe { main_store.static_words_fst(reader)? } {
Some(words) => words, Some(words) => words,
None => return Ok((Vec::new(), 0)), None => return Ok(SortResult::default()),
}; };
let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default(); let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default();
@ -107,6 +120,17 @@ where
docids = Cow::Owned(intersection); docids = Cow::Owned(intersection);
} }
if let Some(facet_count_docids) = facet_count_docids {
let mut facets = HashMap::new();
for (key, document_ids) in facet_count_docids {
let mut counter = Counter::new();
let op = OpBuilder::new(document_ids.as_ref(), document_ids.as_ref()).intersection();
SetOperation::<DocumentId>::extend_collection(op, &mut counter);
facets.insert(key, counter.0);
}
result.facets = Some(facets);
}
let before = Instant::now(); let before = Instant::now();
mk_arena!(arena); mk_arena!(arena);
let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries); let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
@ -181,7 +205,10 @@ where
debug!("bucket sort took {:.02?}", before_bucket_sort.elapsed()); debug!("bucket sort took {:.02?}", before_bucket_sort.elapsed());
Ok((documents, docids.len())) result.documents = documents;
result.nb_hits = docids.len();
Ok(result)
} }
pub fn bucket_sort_with_distinct<'c, FI, FD>( pub fn bucket_sort_with_distinct<'c, FI, FD>(
@ -189,6 +216,7 @@ pub fn bucket_sort_with_distinct<'c, FI, FD>(
query: &str, query: &str,
range: Range<usize>, range: Range<usize>,
facets_docids: Option<SetBuf<DocumentId>>, facets_docids: Option<SetBuf<DocumentId>>,
facet_count_docids: Option<HashMap<FacetKey, Cow<Set<DocumentId>>>>,
filter: Option<FI>, filter: Option<FI>,
distinct: FD, distinct: FD,
distinct_size: usize, distinct_size: usize,
@ -200,14 +228,16 @@ pub fn bucket_sort_with_distinct<'c, FI, FD>(
synonyms_store: store::Synonyms, synonyms_store: store::Synonyms,
_prefix_documents_cache_store: store::PrefixDocumentsCache, _prefix_documents_cache_store: store::PrefixDocumentsCache,
prefix_postings_lists_cache_store: store::PrefixPostingsListsCache, prefix_postings_lists_cache_store: store::PrefixPostingsListsCache,
) -> MResult<(Vec<Document>, usize)> ) -> MResult<SortResult>
where where
FI: Fn(DocumentId) -> bool, FI: Fn(DocumentId) -> bool,
FD: Fn(DocumentId) -> Option<u64>, FD: Fn(DocumentId) -> Option<u64>,
{ {
let mut result = SortResult::default();
let words_set = match unsafe { main_store.static_words_fst(reader)? } { let words_set = match unsafe { main_store.static_words_fst(reader)? } {
Some(words) => words, Some(words) => words,
None => return Ok((Vec::new(), 0)), None => return Ok(SortResult::default()),
}; };
let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default(); let stop_words = main_store.stop_words_fst(reader)?.unwrap_or_default();
@ -240,12 +270,23 @@ where
debug!("number of postings {:?}", queries.len()); debug!("number of postings {:?}", queries.len());
if let Some(facets_docids) = facets_docids { if let Some(facets_docids) = facets_docids {
let intersection = sdset::duo::OpBuilder::new(docids.as_ref(), facets_docids.as_set()) let intersection = OpBuilder::new(docids.as_ref(), facets_docids.as_set())
.intersection() .intersection()
.into_set_buf(); .into_set_buf();
docids = Cow::Owned(intersection); docids = Cow::Owned(intersection);
} }
if let Some(facet_count_docids) = facet_count_docids {
let mut facets = HashMap::new();
for (key, document_ids) in facet_count_docids {
let mut counter = Counter::new();
let op = OpBuilder::new(document_ids.as_ref(), document_ids.as_ref()).intersection();
SetOperation::<DocumentId>::extend_collection(op, &mut counter);
facets.insert(key, counter.0);
}
result.facets = Some(facets);
}
let before = Instant::now(); let before = Instant::now();
mk_arena!(arena); mk_arena!(arena);
let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries); let mut bare_matches = cleanup_bare_matches(&mut arena, &docids, queries);
@ -379,8 +420,10 @@ where
} }
} }
} }
result.documents = documents;
result.nb_hits = docids.len();
Ok((documents, docids.len())) Ok(result)
} }
fn cleanup_bare_matches<'tag, 'txn>( fn cleanup_bare_matches<'tag, 'txn>(

View File

@ -1,27 +1,31 @@
use std::borrow::Cow; use std::borrow::Cow;
use std::collections::HashMap;
use std::ops::{Range, Deref}; use std::ops::{Range, Deref};
use std::time::Duration; use std::time::Duration;
use crate::database::MainT; use crate::database::MainT;
use crate::bucket_sort::{bucket_sort, bucket_sort_with_distinct}; use crate::bucket_sort::{bucket_sort, bucket_sort_with_distinct, SortResult};
use crate::{criterion::Criteria, Document, DocumentId}; use crate::{criterion::Criteria, DocumentId};
use crate::{reordered_attrs::ReorderedAttrs, store, MResult}; use crate::{reordered_attrs::ReorderedAttrs, store, MResult};
use crate::facets::FacetFilter; use crate::facets::FacetFilter;
use either::Either; use either::Either;
use sdset::SetOperation; use sdset::SetOperation;
pub struct QueryBuilder<'c, 'f, 'd, 'fa, 'i> { use meilisearch_schema::FieldId;
pub struct QueryBuilder<'c, 'f, 'd, 'i, 'q> {
criteria: Criteria<'c>, criteria: Criteria<'c>,
searchable_attrs: Option<ReorderedAttrs>, searchable_attrs: Option<ReorderedAttrs>,
filter: Option<Box<dyn Fn(DocumentId) -> bool + 'f>>, filter: Option<Box<dyn Fn(DocumentId) -> bool + 'f>>,
distinct: Option<(Box<dyn Fn(DocumentId) -> Option<u64> + 'd>, usize)>, distinct: Option<(Box<dyn Fn(DocumentId) -> Option<u64> + 'd>, usize)>,
timeout: Option<Duration>, timeout: Option<Duration>,
index: &'i store::Index, index: &'i store::Index,
facets: Option<&'fa FacetFilter>, facet_fitlers: Option<&'q FacetFilter>,
facets: Option<&'q [FieldId]>,
} }
impl<'c, 'f, 'd, 'fa, 'i> QueryBuilder<'c, 'f, 'd, 'fa, 'i> { impl<'c, 'f, 'd, 'i, 'q> QueryBuilder<'c, 'f, 'd, 'i, 'q> {
pub fn new(index: &'i store::Index) -> Self { pub fn new(index: &'i store::Index) -> Self {
QueryBuilder::with_criteria( QueryBuilder::with_criteria(
index, index,
@ -29,7 +33,13 @@ impl<'c, 'f, 'd, 'fa, 'i> QueryBuilder<'c, 'f, 'd, 'fa, 'i> {
) )
} }
pub fn set_facets(&mut self, facets: Option<&'fa FacetFilter>) { /// sets facet attributes to filter on
pub fn set_facet_filters(&mut self, facets: Option<&'q FacetFilter>) {
self.facet_fitlers = facets;
}
/// sets facet attributes for which to return the count
pub fn set_facets(&mut self, facets: Option<&'q [FieldId]>) {
self.facets = facets; self.facets = facets;
} }
@ -44,6 +54,7 @@ impl<'c, 'f, 'd, 'fa, 'i> QueryBuilder<'c, 'f, 'd, 'fa, 'i> {
distinct: None, distinct: None,
timeout: None, timeout: None,
index, index,
facet_fitlers: None,
facets: None, facets: None,
} }
} }
@ -76,8 +87,8 @@ impl<'c, 'f, 'd, 'fa, 'i> QueryBuilder<'c, 'f, 'd, 'fa, 'i> {
reader: &heed::RoTxn<MainT>, reader: &heed::RoTxn<MainT>,
query: &str, query: &str,
range: Range<usize>, range: Range<usize>,
) -> MResult<(Vec<Document>, usize)> { ) -> MResult<SortResult> {
let facets_docids = match self.facets { let facets_docids = match self.facet_fitlers {
Some(facets) => { Some(facets) => {
let mut ands = Vec::with_capacity(facets.len()); let mut ands = Vec::with_capacity(facets.len());
let mut ors = Vec::new(); let mut ors = Vec::new();
@ -98,7 +109,7 @@ impl<'c, 'f, 'd, 'fa, 'i> QueryBuilder<'c, 'f, 'd, 'fa, 'i> {
match self.index.facets.facet_document_ids(reader, &key)? { match self.index.facets.facet_document_ids(reader, &key)? {
Some(docids) => ands.push(docids), Some(docids) => ands.push(docids),
// no candidates for search, early return. // no candidates for search, early return.
None => return Ok((vec![], 0)), None => return Ok(SortResult::default()),
} }
} }
}; };
@ -109,12 +120,29 @@ impl<'c, 'f, 'd, 'fa, 'i> QueryBuilder<'c, 'f, 'd, 'fa, 'i> {
None => None None => None
}; };
let facet_count_docids = match self.facets {
Some(field_ids) => {
let mut facet_count_map = HashMap::new();
for field_id in field_ids {
for pair in self.index.facets.field_document_ids(reader, *field_id)? {
let (facet_key, document_ids) = pair?;
let facet_key_string = facet_key.to_parts(schema)?;
facet_count_map.insert(facet_key, document_ids);
}
}
Some(facet_count_map)
}
None => None,
};
match self.distinct { match self.distinct {
Some((distinct, distinct_size)) => bucket_sort_with_distinct( Some((distinct, distinct_size)) => bucket_sort_with_distinct(
reader, reader,
query, query,
range, range,
facets_docids, facets_docids,
facet_count_docids,
self.filter, self.filter,
distinct, distinct,
distinct_size, distinct_size,
@ -132,6 +160,7 @@ impl<'c, 'f, 'd, 'fa, 'i> QueryBuilder<'c, 'f, 'd, 'fa, 'i> {
query, query,
range, range,
facets_docids, facets_docids,
facet_count_docids,
self.filter, self.filter,
self.criteria, self.criteria,
self.searchable_attrs, self.searchable_attrs,

View File

@ -1,10 +1,11 @@
use std::borrow::Cow; use std::borrow::Cow;
use std::collections::HashMap; use std::collections::HashMap;
use heed::{RwTxn, RoTxn, Result as ZResult}; use heed::{RwTxn, RoTxn, Result as ZResult, RoRange};
use sdset::{SetBuf, Set, SetOperation}; use sdset::{SetBuf, Set, SetOperation};
use meilisearch_types::DocumentId; use meilisearch_types::DocumentId;
use meilisearch_schema::FieldId;
use crate::database::MainT; use crate::database::MainT;
use crate::facets::FacetKey; use crate::facets::FacetKey;
@ -22,6 +23,10 @@ impl Facets {
self.facets.put(writer, &facet_key, doc_ids) self.facets.put(writer, &facet_key, doc_ids)
} }
pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, field_id: FieldId) -> ZResult<RoRange<'txn, FacetKey, CowSet<DocumentId>>> {
self.facets.prefix_iter(reader, &FacetKey::new(field_id, "".to_string()))
}
pub fn facet_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, facet_key: &FacetKey) -> ZResult<Option<Cow<'txn, Set<DocumentId>>>> { pub fn facet_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, facet_key: &FacetKey) -> ZResult<Option<Cow<'txn, Set<DocumentId>>>> {
self.facets.get(reader, &facet_key) self.facets.get(reader, &facet_key)
} }

View File

@ -363,10 +363,10 @@ impl Index {
QueryBuilder::new(self) QueryBuilder::new(self)
} }
pub fn query_builder_with_criteria<'c, 'f, 'd, 'fa, 'i>( pub fn query_builder_with_criteria<'c, 'f, 'd, 'fa, 'i, 'q>(
&'i self, &'i self,
criteria: Criteria<'c>, criteria: Criteria<'c>,
) -> QueryBuilder<'c, 'f, 'd, 'fa, 'i> { ) -> QueryBuilder<'c, 'f, 'd, 'i, 'q> {
QueryBuilder::with_criteria(self, criteria) QueryBuilder::with_criteria(self, criteria)
} }
} }

View File

@ -36,6 +36,7 @@ impl IndexSearchExt for Index {
filters: None, filters: None,
matches: false, matches: false,
facet_filters: None, facet_filters: None,
facets: None,
} }
} }
} }
@ -51,6 +52,7 @@ pub struct SearchBuilder<'a> {
filters: Option<String>, filters: Option<String>,
matches: bool, matches: bool,
facet_filters: Option<FacetFilter>, facet_filters: Option<FacetFilter>,
facets: Option<Vec<FieldId>>
} }
impl<'a> SearchBuilder<'a> { impl<'a> SearchBuilder<'a> {
@ -100,6 +102,11 @@ impl<'a> SearchBuilder<'a> {
self self
} }
pub fn add_facets(&mut self, facets: Vec<FieldId>) -> &SearchBuilder {
self.facets = Some(facets);
self
}
pub fn search(&self, reader: &heed::RoTxn<MainT>) -> Result<SearchResult, ResponseError> { pub fn search(&self, reader: &heed::RoTxn<MainT>) -> Result<SearchResult, ResponseError> {
let schema = self let schema = self
.index .index
@ -146,11 +153,12 @@ impl<'a> SearchBuilder<'a> {
} }
} }
query_builder.set_facets(self.facet_filters.as_ref()); query_builder.set_facet_filters(self.facet_filters.as_ref());
query_builder.set_facets(self.facets.as_deref());
let start = Instant::now(); let start = Instant::now();
let result = query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit)); let result = query_builder.query(reader, &self.query, self.offset..(self.offset + self.limit));
let (docs, nb_hits) = result.map_err(ResponseError::search_documents)?; let search_result = result.map_err(ResponseError::search_documents)?;
let time_ms = start.elapsed().as_millis() as usize; let time_ms = start.elapsed().as_millis() as usize;
let mut all_attributes: HashSet<&str> = HashSet::new(); let mut all_attributes: HashSet<&str> = HashSet::new();
@ -181,7 +189,7 @@ impl<'a> SearchBuilder<'a> {
} }
let mut hits = Vec::with_capacity(self.limit); let mut hits = Vec::with_capacity(self.limit);
for doc in docs { for doc in search_result.documents {
let mut document: IndexMap<String, Value> = self let mut document: IndexMap<String, Value> = self
.index .index
.document(reader, Some(&all_attributes), doc.id) .document(reader, Some(&all_attributes), doc.id)
@ -235,10 +243,11 @@ impl<'a> SearchBuilder<'a> {
hits, hits,
offset: self.offset, offset: self.offset,
limit: self.limit, limit: self.limit,
nb_hits, nb_hits: search_result.nb_hits,
exhaustive_nb_hits: false, exhaustive_nb_hits: search_result.is_exhaustive,
processing_time_ms: time_ms, processing_time_ms: time_ms,
query: self.query.to_string(), query: self.query.to_string(),
facets: search_result.facets
}; };
Ok(results) Ok(results)

View File

@ -5,6 +5,7 @@ use actix_web::web;
use actix_web::HttpResponse; use actix_web::HttpResponse;
use actix_web_macros::get; use actix_web_macros::get;
use serde::Deserialize; use serde::Deserialize;
use serde_json::Value;
use crate::error::ResponseError; use crate::error::ResponseError;
use crate::helpers::meilisearch::IndexSearchExt; use crate::helpers::meilisearch::IndexSearchExt;
@ -13,6 +14,7 @@ use crate::routes::IndexParam;
use crate::Data; use crate::Data;
use meilisearch_core::facets::FacetFilter; use meilisearch_core::facets::FacetFilter;
use meilisearch_schema::{Schema, FieldId};
pub fn services(cfg: &mut web::ServiceConfig) { pub fn services(cfg: &mut web::ServiceConfig) {
cfg.service(search_with_url_query); cfg.service(search_with_url_query);
@ -31,6 +33,7 @@ struct SearchQuery {
filters: Option<String>, filters: Option<String>,
matches: Option<bool>, matches: Option<bool>,
facet_filters: Option<String>, facet_filters: Option<String>,
facets: Option<String>
} }
#[get("/indexes/{index_uid}/search", wrap = "Authentication::Public")] #[get("/indexes/{index_uid}/search", wrap = "Authentication::Public")]
@ -91,6 +94,14 @@ async fn search_with_url_query(
} }
} }
if let Some(ref facets) = params.facets {
match index.main.attributes_for_faceting(&reader)? {
Some(ref attrs) => { search_builder.add_facets(prepare_facet_list(facets, &schema, attrs)?); },
None => return Err(ResponseError::FacetExpression("can't return facets count, as no facet is set".to_string()))
}
}
if let Some(attributes_to_crop) = &params.attributes_to_crop { if let Some(attributes_to_crop) = &params.attributes_to_crop {
let default_length = params.crop_length.unwrap_or(200); let default_length = params.crop_length.unwrap_or(200);
let mut final_attributes: HashMap<String, usize> = HashMap::new(); let mut final_attributes: HashMap<String, usize> = HashMap::new();
@ -150,3 +161,27 @@ async fn search_with_url_query(
Ok(HttpResponse::Ok().json(search_builder.search(&reader)?)) Ok(HttpResponse::Ok().json(search_builder.search(&reader)?))
} }
fn prepare_facet_list<'fa>(facets: &str, schema: &Schema, facet_attrs: &'fa [FieldId]) -> Result<Vec<FieldId>, ResponseError> {
let facet_array = serde_json::from_str(facets).expect("do error handling"); // TODO
match facet_array {
Value::Array(facet_array) => {
let wild_card = Value::String("*".to_string());
if facet_array.iter().any(|it| it == &wild_card) {
return Ok(Vec::from(facet_attrs)); // TODO can make cow?
}
let mut fields = Vec::with_capacity(facet_attrs.len());
for v in facet_array {
match v {
Value::String(name) => {
let id = schema.id(&name).expect("not found error"); // TODO
fields.push(id);
}
_ => todo!("expected string, found {}", v),
}
}
return Ok(fields);
}
_ => todo!("error, bad syntax, expected array")
}
}