snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-01-19 01:18:31 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

get back to what we were doingb efore in the DB cache and with the restricted field id

Browse Source
This commit is contained in:
Tamo 2024-05-15 18:00:39 +02:00
parent ad4d8502b3
commit 5542f1d9f1
2 changed files with 99 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
milli/src/search/new/db_cache.rs
View File

@ -159,36 +159,58 @@ impl<'ctx> SearchContext<'ctx> {
/// Retrieve or insert the given value in the `word_docids` database. /// Retrieve or insert the given value in the `word_docids` database.
fn get_db_word_docids(&mut self, word: Interned<String>) -> Result<Option<RoaringBitmap>> { fn get_db_word_docids(&mut self, word: Interned<String>) -> Result<Option<RoaringBitmap>> {
let interned = self.word_interner.get(word).as_str(); match &self.restricted_fids {
let keys: Vec<_> = Some(restricted_fids) => {
self.searchable_fids.tolerant.iter().map(|(fid, _weight)| (interned, *fid)).collect(); let interned = self.word_interner.get(word).as_str();
let keys: Vec<_> =
restricted_fids.tolerant.iter().map(|(fid, _)| (interned, *fid)).collect();
DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
word, word,
&keys[..], &keys[..],
&mut self.db_cache.word_docids, &mut self.db_cache.word_docids,
self.index.word_fid_docids.remap_data_type::<Bytes>(), self.index.word_fid_docids.remap_data_type::<Bytes>(),
merge_cbo_roaring_bitmaps, merge_cbo_roaring_bitmaps,
) )
}
None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn,
word,
self.word_interner.get(word).as_str(),
&mut self.db_cache.word_docids,
self.index.word_docids.remap_data_type::<Bytes>(),
),
}
} }
fn get_db_exact_word_docids( fn get_db_exact_word_docids(
&mut self, &mut self,
word: Interned<String>, word: Interned<String>,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
let interned = self.word_interner.get(word).as_str(); match &self.restricted_fids {
let keys: Vec<_> = Some(restricted_fids) => {
self.searchable_fids.exact.iter().map(|(fid, _weight)| (interned, *fid)).collect(); let interned = self.word_interner.get(word).as_str();
let keys: Vec<_> =
restricted_fids.exact.iter().map(|(fid, _)| (interned, *fid)).collect();
DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
word, word,
&keys[..], &keys[..],
&mut self.db_cache.exact_word_docids, &mut self.db_cache.exact_word_docids,
self.index.word_fid_docids.remap_data_type::<Bytes>(), self.index.word_fid_docids.remap_data_type::<Bytes>(),
merge_cbo_roaring_bitmaps, merge_cbo_roaring_bitmaps,
) )
}
None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn,
word,
self.word_interner.get(word).as_str(),
&mut self.db_cache.exact_word_docids,
self.index.exact_word_docids.remap_data_type::<Bytes>(),
),
}
} }
pub fn word_prefix_docids(&mut self, prefix: Word) -> Result<Option<RoaringBitmap>> { pub fn word_prefix_docids(&mut self, prefix: Word) -> Result<Option<RoaringBitmap>> {
@ -216,36 +238,58 @@ impl<'ctx> SearchContext<'ctx> {
&mut self, &mut self,
prefix: Interned<String>, prefix: Interned<String>,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
let interned = self.word_interner.get(prefix).as_str(); match &self.restricted_fids {
let keys: Vec<_> = Some(restricted_fids) => {
self.searchable_fids.tolerant.iter().map(|(fid, _weight)| (interned, *fid)).collect(); let interned = self.word_interner.get(prefix).as_str();
let keys: Vec<_> =
restricted_fids.tolerant.iter().map(|(fid, _)| (interned, *fid)).collect();
DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
prefix, prefix,
&keys[..], &keys[..],
&mut self.db_cache.word_prefix_docids, &mut self.db_cache.word_prefix_docids,
self.index.word_prefix_fid_docids.remap_data_type::<Bytes>(), self.index.word_prefix_fid_docids.remap_data_type::<Bytes>(),
merge_cbo_roaring_bitmaps, merge_cbo_roaring_bitmaps,
) )
}
None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn,
prefix,
self.word_interner.get(prefix).as_str(),
&mut self.db_cache.word_prefix_docids,
self.index.word_prefix_docids.remap_data_type::<Bytes>(),
),
}
} }
fn get_db_exact_word_prefix_docids( fn get_db_exact_word_prefix_docids(
&mut self, &mut self,
prefix: Interned<String>, prefix: Interned<String>,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
let interned = self.word_interner.get(prefix).as_str(); match &self.restricted_fids {
let keys: Vec<_> = Some(restricted_fids) => {
self.searchable_fids.exact.iter().map(|(fid, _weight)| (interned, *fid)).collect(); let interned = self.word_interner.get(prefix).as_str();
let keys: Vec<_> =
restricted_fids.exact.iter().map(|(fid, _)| (interned, *fid)).collect();
DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
prefix, prefix,
&keys[..], &keys[..],
&mut self.db_cache.exact_word_prefix_docids, &mut self.db_cache.exact_word_prefix_docids,
self.index.word_prefix_fid_docids.remap_data_type::<Bytes>(), self.index.word_prefix_fid_docids.remap_data_type::<Bytes>(),
merge_cbo_roaring_bitmaps, merge_cbo_roaring_bitmaps,
) )
}
None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn,
prefix,
self.word_interner.get(prefix).as_str(),
&mut self.db_cache.exact_word_prefix_docids,
self.index.exact_word_prefix_docids.remap_data_type::<Bytes>(),
),
}
} }
pub fn get_db_word_pair_proximity_docids( pub fn get_db_word_pair_proximity_docids(
@ -421,8 +465,8 @@ impl<'ctx> SearchContext<'ctx> {
word: Interned<String>, word: Interned<String>,
fid: u16, fid: u16,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
// if the requested fid isn't in the list of searchable, return None. // if the requested fid isn't in the restricted list, return None.
if !self.searchable_fids.contains(&fid) { if self.restricted_fids.as_ref().map_or(false, |fids| !fids.contains(&fid)) {
return Ok(None); return Ok(None);
} }
@ -440,8 +484,8 @@ impl<'ctx> SearchContext<'ctx> {
word_prefix: Interned<String>, word_prefix: Interned<String>,
fid: u16, fid: u16,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
// if the requested fid isn't in the searchable list, return None. // if the requested fid isn't in the restricted list, return None.
if !self.searchable_fids.contains(&fid) { if self.restricted_fids.as_ref().map_or(false, |fids| !fids.contains(&fid)) {
return Ok(None); return Ok(None);
} }

19
milli/src/search/new/mod.rs
View File

@ -66,7 +66,7 @@ pub struct SearchContext<'ctx> {
pub phrase_interner: DedupInterner<Phrase>, pub phrase_interner: DedupInterner<Phrase>,
pub term_interner: Interner<QueryTerm>, pub term_interner: Interner<QueryTerm>,
pub phrase_docids: PhraseDocIdsCache, pub phrase_docids: PhraseDocIdsCache,
pub searchable_fids: SearchableFids, pub restricted_fids: Option<RestrictedFids>,
} }
impl<'ctx> SearchContext<'ctx> { impl<'ctx> SearchContext<'ctx> {
@ -92,7 +92,7 @@ impl<'ctx> SearchContext<'ctx> {
phrase_interner: <_>::default(), phrase_interner: <_>::default(),
term_interner: <_>::default(), term_interner: <_>::default(),
phrase_docids: <_>::default(), phrase_docids: <_>::default(),
searchable_fids: SearchableFids { tolerant, exact }, restricted_fids: None,
}) })
} }
@ -103,7 +103,7 @@ impl<'ctx> SearchContext<'ctx> {
let mut wildcard = false; let mut wildcard = false;
let mut restricted_fids = SearchableFids::default(); let mut restricted_fids = RestrictedFids::default();
for field_name in attributes_to_search_on { for field_name in attributes_to_search_on {
if field_name == "*" { if field_name == "*" {
wildcard = true; wildcard = true;
@ -141,14 +141,9 @@ impl<'ctx> SearchContext<'ctx> {
} }
if wildcard { if wildcard {
let (exact, tolerant) = searchable_names self.restricted_fids = None;
.iter()
.map(|(_name, fid, weight)| (*fid, *weight))
.partition(|(fid, _weight)| exact_attributes_ids.contains(fid));
self.searchable_fids = SearchableFids { tolerant, exact };
} else { } else {
self.searchable_fids = restricted_fids; self.restricted_fids = Some(restricted_fids);
} }
Ok(()) Ok(())
@ -171,12 +166,12 @@ impl Word {
} }
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct SearchableFids { pub struct RestrictedFids {
pub tolerant: Vec<(FieldId, Weight)>, pub tolerant: Vec<(FieldId, Weight)>,
pub exact: Vec<(FieldId, Weight)>, pub exact: Vec<(FieldId, Weight)>,
} }
impl SearchableFids { impl RestrictedFids {
pub fn contains(&self, fid: &FieldId) -> bool { pub fn contains(&self, fid: &FieldId) -> bool {
self.tolerant.iter().any(|(id, _)| id == fid) || self.exact.iter().any(|(id, _)| id == fid) self.tolerant.iter().any(|(id, _)| id == fid) || self.exact.iter().any(|(id, _)| id == fid)
} }