snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-30 09:04:59 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Limit the number of derivations for a single word.

Browse Source
This commit is contained in:
Louis Dureuil 2023-03-31 09:19:18 +02:00
parent c2b025946a
commit 9b87c36200
3 changed files with 63 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
milli/src/search/new/limits.rs Normal file
View File

@ -0,0 +1,18 @@
/// Maximum number of tokens we consider in a single search.
// TODO: Loic, find proper value here so we don't overflow the interner.
pub const MAX_TOKEN_COUNT: usize = 1_000;
/// Maximum number of prefixes that can be derived from a single word.
pub const MAX_PREFIX_COUNT: usize = 1_000;
/// Maximum number of words that can be derived from a single word with a distance of one to that word.
pub const MAX_ONE_TYPO_COUNT: usize = 150;
/// Maximum number of words that can be derived from a single word with a distance of two to that word.
pub const MAX_TWO_TYPOS_COUNT: usize = 50;
/// Maximum amount of synonym phrases that can be derived from a single word.
pub const MAX_SYNONYM_PHRASE_COUNT: usize = 50;
/// Maximum amount of words inside of all the synonym phrases that can be derived from a single word.
///
/// This limit is meant to gracefully handle the case where a word would have very long phrases as synonyms.
pub const MAX_SYNONYM_WORD_COUNT: usize = 100;

1
milli/src/search/new/mod.rs
View File

@ -2,6 +2,7 @@ mod db_cache;
mod distinct;
mod graph_based_ranking_rule;
mod interner;
mod limits;
mod logger;
mod query_graph;
mod query_term;

56
milli/src/search/new/query_term.rs
View File

@ -10,7 +10,7 @@ use heed::RoTxn;
use itertools::Itertools;
use super::interner::{DedupInterner, Interned};
use super::SearchContext;
use super::{limits, SearchContext};
use crate::search::fst_utils::{Complement, Intersection, StartsWith, Union};
use crate::search::{build_dfa, get_first};
use crate::{CboRoaringBitmapLenCodec, Index, Result, MAX_WORD_LENGTH};
@ -266,6 +266,9 @@ pub fn query_term_from_word(
let mut stream = fst.search(prefix).into_stream();
while let Some(derived_word) = stream.next() {
if prefix_of.len() >= limits::MAX_PREFIX_COUNT {
break;
}
let derived_word = std::str::from_utf8(derived_word)?.to_owned();
let derived_word_interned = ctx.word_interner.insert(derived_word);
if derived_word_interned != word_interned {
@ -277,23 +280,31 @@ pub fn query_term_from_word(
let dfa = build_dfa(word, 1, is_prefix);
let starts = StartsWith(Str::new(get_first(word)));
let mut stream = fst.search_with_state(Intersection(starts, &dfa)).into_stream();
// TODO: There may be wayyy too many matches (e.g. in the thousands), how to reduce them?
while let Some((derived_word, state)) = stream.next() {
if prefix_of.len() >= limits::MAX_PREFIX_COUNT
&& one_typo.len() >= limits::MAX_ONE_TYPO_COUNT
{
break;
}
let derived_word = std::str::from_utf8(derived_word)?;
let d = dfa.distance(state.1);
let derived_word_interned = ctx.word_interner.insert(derived_word.to_owned());
match d.to_u8() {
0 => {
if derived_word_interned != word_interned {
if derived_word_interned != word_interned
&& prefix_of.len() < limits::MAX_PREFIX_COUNT
{
prefix_of.push(derived_word_interned);
}
}
1 => {
if one_typo.len() < limits::MAX_PREFIX_COUNT {
one_typo.push(derived_word_interned);
}
_ => panic!(),
}
_ => unreachable!("One typo dfa produced multiple typos"),
}
}
} else {
@ -304,14 +315,21 @@ pub fn query_term_from_word(
let automaton = Union(first, &second);
let mut stream = fst.search_with_state(automaton).into_stream();
// TODO: There may be wayyy too many matches (e.g. in the thousands), how to reduce them?
while let Some((derived_word, state)) = stream.next() {
if prefix_of.len() >= limits::MAX_PREFIX_COUNT
&& one_typo.len() >= limits::MAX_ONE_TYPO_COUNT
&& two_typos.len() >= limits::MAX_TWO_TYPOS_COUNT
{
break;
}
let derived_word = std::str::from_utf8(derived_word)?;
let derived_word_interned = ctx.word_interner.insert(derived_word.to_owned());
// in the case the typo is on the first letter, we know the number of typo
// is two
if get_first(derived_word) != get_first(word) {
if get_first(derived_word) != get_first(word)
&& two_typos.len() < limits::MAX_TWO_TYPOS_COUNT
{
two_typos.push(derived_word_interned);
} else {
// Else, we know that it is the second dfa that matched and compute the
@ -319,17 +337,23 @@ pub fn query_term_from_word(
let d = second_dfa.distance((state.1).0);
match d.to_u8() {
0 => {
if derived_word_interned != word_interned {
if derived_word_interned != word_interned
&& prefix_of.len() < limits::MAX_PREFIX_COUNT
{
prefix_of.push(derived_word_interned);
}
}
1 => {
if one_typo.len() < limits::MAX_ONE_TYPO_COUNT {
one_typo.push(derived_word_interned);
}
}
2 => {
if two_typos.len() < limits::MAX_TWO_TYPOS_COUNT {
two_typos.push(derived_word_interned);
}
_ => panic!(),
}
_ => unreachable!("2 typos DFA produced a distance greater than 2"),
}
}
}
@ -341,15 +365,20 @@ pub fn query_term_from_word(
});
let synonyms = ctx.index.synonyms(ctx.txn)?;
let mut synonym_word_count = 0;
let synonyms = synonyms
.get(&vec![word.to_owned()])
.cloned()
.unwrap_or_default()
.into_iter()
.map(|words| {
.take(limits::MAX_SYNONYM_PHRASE_COUNT)
.filter_map(|words| {
if synonym_word_count + words.len() > limits::MAX_SYNONYM_WORD_COUNT {
return None;
}
synonym_word_count += words.len();
let words = words.into_iter().map(|w| Some(ctx.word_interner.insert(w))).collect();
ctx.phrase_interner.insert(Phrase { words })
Some(ctx.phrase_interner.insert(Phrase { words }))
})
.collect();
@ -488,10 +517,7 @@ pub fn located_query_terms_from_string(
// start with the last position as we will wrap around to position 0 at the beginning of the loop below.
let mut position = u16::MAX;
// TODO: Loic, find proper value here so we don't overflow the interner.
const MAX_TOKEN_COUNT: usize = 1_000;
let mut peekable = query.take(MAX_TOKEN_COUNT).peekable();
let mut peekable = query.take(super::limits::MAX_TOKEN_COUNT).peekable();
while let Some(token) = peekable.next() {
// early return if word limit is exceeded
if located_terms.len() >= parts_limit {