Merge #596

596: Filter operators: NOT + IN[..] r=irevoire a=loiclec

# Pull Request

## What does this PR do?
Implements the changes described in https://github.com/meilisearch/meilisearch/issues/2580
It is based on top of #556 

Co-authored-by: Loïc Lecrenier <loic@meilisearch.com>

filter-parser/Cargo.toml

@@ -8,3 +8,6 @@ publish = false
 [dependencies]
 nom = "7.1.0"
 nom_locate = "4.0.0"
+
+[dev-dependencies]
+insta = "1.18.2"

filter-parser/src/condition.rs

@@ -21,30 +21,12 @@ pub enum Condition<'a> {
     Equal(Token<'a>),
     NotEqual(Token<'a>),
     Exists,
-    NotExists,
     LowerThan(Token<'a>),
     LowerThanOrEqual(Token<'a>),
     Between { from: Token<'a>, to: Token<'a> },
 }
 
-impl<'a> Condition<'a> {
+/// condition      = value ("==" | ">" ...) value
-    /// This method can return two operations in case it must express
-    /// an OR operation for the between case (i.e. `TO`).
-    pub fn negate(self) -> (Self, Option<Self>) {
-        match self {
-            GreaterThan(n) => (LowerThanOrEqual(n), None),
-            GreaterThanOrEqual(n) => (LowerThan(n), None),
-            Equal(s) => (NotEqual(s), None),
-            NotEqual(s) => (Equal(s), None),
-            Exists => (NotExists, None),
-            NotExists => (Exists, None),
-            LowerThan(n) => (GreaterThanOrEqual(n), None),
-            LowerThanOrEqual(n) => (GreaterThan(n), None),
-            Between { from, to } => (LowerThan(from), Some(GreaterThan(to))),
-        }
-    }
-}
-/// condition      = value ("=" | "!=" | ">" | ">=" | "<" | "<=") value
 pub fn parse_condition(input: Span) -> IResult<FilterCondition> {
     let operator = alt((tag("<="), tag(">="), tag("!="), tag("<"), tag(">"), tag("=")));
     let (input, (fid, op, value)) = tuple((parse_value, operator, cut(parse_value)))(input)?;
@@ -73,7 +55,10 @@ pub fn parse_not_exists(input: Span) -> IResult<FilterCondition> {
     let (input, key) = parse_value(input)?;
 
     let (input, _) = tuple((tag("NOT"), multispace1, tag("EXISTS")))(input)?;
-    Ok((input, FilterCondition::Condition { fid: key.into(), op: NotExists }))
+    Ok((
+        input,
+        FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key.into(), op: Exists })),
+    ))
 }
 
 /// to             = value value "TO" WS+ value

filter-parser/src/error.rs

@@ -48,6 +48,12 @@ pub struct Error<'a> {
     kind: ErrorKind<'a>,
 }
 
+#[derive(Debug)]
+pub enum ExpectedValueKind {
+    ReservedKeyword,
+    Other,
+}
+
 #[derive(Debug)]
 pub enum ErrorKind<'a> {
     ReservedGeo(&'a str),
@@ -55,11 +61,16 @@ pub enum ErrorKind<'a> {
     MisusedGeo,
     InvalidPrimary,
     ExpectedEof,
-    ExpectedValue,
+    ExpectedValue(ExpectedValueKind),
     MalformedValue,
+    InOpeningBracket,
+    InClosingBracket,
+    InExpectedValue(ExpectedValueKind),
+    ReservedKeyword(String),
     MissingClosingDelimiter(char),
     Char(char),
     InternalError(error::ErrorKind),
+    DepthLimitReached,
     External(String),
 }
 
@@ -109,24 +120,26 @@ impl<'a> ParseError<Span<'a>> for Error<'a> {
 impl<'a> Display for Error<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let input = self.context.fragment();
-
         // When printing our error message we want to escape all `\n` to be sure we keep our format with the
         // first line being the diagnostic and the second line being the incriminated filter.
         let escaped_input = input.escape_debug();
 
-        match self.kind {
+        match &self.kind {
-            ErrorKind::ExpectedValue if input.trim().is_empty() => {
+            ErrorKind::ExpectedValue(_) if input.trim().is_empty() => {
                 writeln!(f, "Was expecting a value but instead got nothing.")?
             }
+            ErrorKind::ExpectedValue(ExpectedValueKind::ReservedKeyword) => {
+                writeln!(f, "Was expecting a value but instead got `{escaped_input}`, which is a reserved keyword. To use `{escaped_input}` as a field name or a value, surround it by quotes.")?
+            }
+            ErrorKind::ExpectedValue(ExpectedValueKind::Other) => {
+                writeln!(f, "Was expecting a value but instead got `{}`.", escaped_input)?
+            }
             ErrorKind::MalformedValue => {
                 writeln!(f, "Malformed value: `{}`.", escaped_input)?
             }
             ErrorKind::MissingClosingDelimiter(c) => {
                 writeln!(f, "Expression `{}` is missing the following closing delimiter: `{}`.", escaped_input, c)?
             }
-            ErrorKind::ExpectedValue => {
-                writeln!(f, "Was expecting a value but instead got `{}`.", escaped_input)?
-            }
             ErrorKind::InvalidPrimary if input.trim().is_empty() => {
                 writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `TO`, `EXISTS`, `NOT EXISTS`, or `_geoRadius` but instead got nothing.")?
             }
@@ -145,9 +158,28 @@ impl<'a> Display for Error<'a> {
             ErrorKind::MisusedGeo => {
                 writeln!(f, "The `_geoRadius` filter is an operation and can't be used as a value.")?
             }
+            ErrorKind::ReservedKeyword(word) => {
+                writeln!(f, "`{word}` is a reserved keyword and thus cannot be used as a field name unless it is put inside quotes. Use \"{word}\" or \'{word}\' instead.")?
+            }
+            ErrorKind::InOpeningBracket => {
+                writeln!(f, "Expected `[` after `IN` keyword.")?
+            }
+            ErrorKind::InClosingBracket => {
+                writeln!(f, "Expected matching `]` after the list of field names given to `IN[`")?
+            }
+            ErrorKind::InExpectedValue(ExpectedValueKind::ReservedKeyword) => {
+                writeln!(f, "Expected only comma-separated field names inside `IN[..]` but instead found `{escaped_input}`, which is a keyword. To use `{escaped_input}` as a field name or a value, surround it by quotes.")?
+            }
+            ErrorKind::InExpectedValue(ExpectedValueKind::Other) => {
+                writeln!(f, "Expected only comma-separated field names inside `IN[..]` but instead found `{escaped_input}`.")?
+            }
             ErrorKind::Char(c) => {
                 panic!("Tried to display a char error with `{}`", c)
             }
+            ErrorKind::DepthLimitReached => writeln!(
+                f,
+                "The filter exceeded the maximum depth limit. Try rewriting the filter so that it contains fewer nested conditions."
+            )?,
             ErrorKind::InternalError(kind) => writeln!(
                 f,
                 "Encountered an internal `{:?}` error while parsing your filter. Please fill an issue", kind

filter-parser/src/value.rs

@@ -5,7 +5,7 @@ use nom::combinator::cut;
 use nom::sequence::{delimited, terminated};
 use nom::{InputIter, InputLength, InputTake, Slice};
 
-use crate::error::NomErrorExt;
+use crate::error::{ExpectedValueKind, NomErrorExt};
 use crate::{parse_geo_point, parse_geo_radius, Error, ErrorKind, IResult, Span, Token};
 
 /// This function goes through all characters in the [Span] if it finds any escaped character (`\`).
@@ -48,6 +48,35 @@ fn quoted_by(quote: char, input: Span) -> IResult<Token> {
     ))
 }
 
+// word           = (alphanumeric | _ | - | .)+    except for reserved keywords
+pub fn word_not_keyword<'a>(input: Span<'a>) -> IResult<Token<'a>> {
+    let (input, word): (_, Token<'a>) =
+        take_while1(is_value_component)(input).map(|(s, t)| (s, t.into()))?;
+    if is_keyword(word.value()) {
+        return Err(nom::Err::Error(Error::new_from_kind(
+            input,
+            ErrorKind::ReservedKeyword(word.value().to_owned()),
+        )));
+    }
+    Ok((input, word))
+}
+
+// word           = {tag}
+pub fn word_exact<'a, 'b: 'a>(tag: &'b str) -> impl Fn(Span<'a>) -> IResult<'a, Token<'a>> {
+    move |input| {
+        let (input, word): (_, Token<'a>) =
+            take_while1(is_value_component)(input).map(|(s, t)| (s, t.into()))?;
+        if word.value() == tag {
+            Ok((input, word))
+        } else {
+            Err(nom::Err::Error(Error::new_from_kind(
+                input,
+                ErrorKind::InternalError(nom::error::ErrorKind::Tag),
+            )))
+        }
+    }
+}
+
 /// value          = WS* ( word | singleQuoted | doubleQuoted) WS+
 pub fn parse_value<'a>(input: Span<'a>) -> IResult<Token<'a>> {
     // to get better diagnostic message we are going to strip the left whitespaces from the input right now
@@ -71,11 +100,6 @@ pub fn parse_value<'a>(input: Span<'a>) -> IResult<Token<'a>> {
         _ => (),
     }
 
-    // word           = (alphanumeric | _ | - | .)+
-    let word = |input: Span<'a>| -> IResult<Token<'a>> {
-        take_while1(is_value_component)(input).map(|(s, t)| (s, t.into()))
-    };
-
     // this parser is only used when an error is encountered and it parse the
     // largest string possible that do not contain any “language” syntax.
     // If we try to parse `name = 🦀 AND language = rust` we want to return an
@@ -85,17 +109,27 @@ pub fn parse_value<'a>(input: Span<'a>) -> IResult<Token<'a>> {
     // when we create the errors from the output of the alt we have spaces everywhere
     let error_word = take_till::<_, _, Error>(is_syntax_component);
 
-    terminated(
+    let (input, value) = terminated(
         alt((
             delimited(char('\''), cut(|input| quoted_by('\'', input)), cut(char('\''))),
             delimited(char('"'), cut(|input| quoted_by('"', input)), cut(char('"'))),
-            word,
+            word_not_keyword,
         )),
         multispace0,
     )(input)
     // if we found nothing in the alt it means the user specified something that was not recognized as a value
     .map_err(|e: nom::Err<Error>| {
-        e.map_err(|_| Error::new_from_kind(error_word(input).unwrap().1, ErrorKind::ExpectedValue))
+        e.map_err(|error| {
+            let expected_value_kind = if matches!(error.kind(), ErrorKind::ReservedKeyword(_)) {
+                ExpectedValueKind::ReservedKeyword
+            } else {
+                ExpectedValueKind::Other
+            };
+            Error::new_from_kind(
+                error_word(input).unwrap().1,
+                ErrorKind::ExpectedValue(expected_value_kind),
+            )
+        })
     })
     .map_err(|e| {
         e.map_fail(|failure| {
@@ -107,7 +141,9 @@ pub fn parse_value<'a>(input: Span<'a>) -> IResult<Token<'a>> {
                 failure
             }
         })
-    })
+    })?;
+
+    Ok((input, value))
 }
 
 fn is_value_component(c: char) -> bool {
@@ -118,6 +154,10 @@ fn is_syntax_component(c: char) -> bool {
     c.is_whitespace() || ['(', ')', '=', '<', '>', '!'].contains(&c)
 }
 
+fn is_keyword(s: &str) -> bool {
+    matches!(s, "AND" | "OR" | "IN" | "NOT" | "TO" | "EXISTS" | "_geoRadius")
+}
+
 #[cfg(test)]
 pub mod test {
     use nom::Finish;

http-ui/src/main.rs

@@ -744,10 +744,9 @@ async fn main() -> anyhow::Result<()> {
             };
 
             let condition = match (filters, facet_filters) {
-                (Some(filters), Some(facet_filters)) => Some(FilterCondition::And(
+                (Some(filters), Some(facet_filters)) => {
-                    Box::new(filters.into()),
+                    Some(FilterCondition::And(vec![filters.into(), facet_filters.into()]))
-                    Box::new(facet_filters.into()),
+                }
-                )),
                 (Some(condition), None) | (None, Some(condition)) => Some(condition.into()),
                 _otherwise => None,
             };

milli/src/search/facet/filter.rs

@@ -89,52 +89,44 @@ impl<'a> Filter<'a> {
         I: IntoIterator<Item = Either<J, &'a str>>,
         J: IntoIterator<Item = &'a str>,
     {
-        let mut ands: Option<FilterCondition> = None;
+        let mut ands = vec![];
 
         for either in array {
             match either {
                 Either::Left(array) => {
-                    let mut ors = None;
+                    let mut ors = vec![];
                     for rule in array {
                         if let Some(filter) = Self::from_str(rule.as_ref())? {
-                            let condition = filter.condition;
+                            ors.push(filter.condition);
-                            ors = match ors.take() {
-                                Some(ors) => {
-                                    Some(FilterCondition::Or(Box::new(ors), Box::new(condition)))
-                                }
-                                None => Some(condition),
-                            };
                         }
                     }
 
-                    if let Some(rule) = ors {
+                    if ors.len() > 1 {
-                        ands = match ands.take() {
+                        ands.push(FilterCondition::Or(ors));
-                            Some(ands) => {
+                    } else if ors.len() == 1 {
-                                Some(FilterCondition::And(Box::new(ands), Box::new(rule)))
+                        ands.push(ors.pop().unwrap());
-                            }
-                            None => Some(rule),
-                        };
                     }
                 }
                 Either::Right(rule) => {
                     if let Some(filter) = Self::from_str(rule.as_ref())? {
-                        let condition = filter.condition;
+                        ands.push(filter.condition);
-                        ands = match ands.take() {
-                            Some(ands) => {
-                                Some(FilterCondition::And(Box::new(ands), Box::new(condition)))
-                            }
-                            None => Some(condition),
-                        };
                     }
                 }
             }
         }
+        let and = if ands.is_empty() {
+            return Ok(None);
+        } else if ands.len() == 1 {
+            ands.pop().unwrap()
+        } else {
+            FilterCondition::And(ands)
+        };
 
-        if let Some(token) = ands.as_ref().and_then(|fc| fc.token_at_depth(MAX_FILTER_DEPTH)) {
+        if let Some(token) = and.token_at_depth(MAX_FILTER_DEPTH) {
             return Err(token.as_external_error(FilterError::TooDeep).into());
         }
 
-        Ok(ands.map(|ands| Self { condition: ands }))
+        Ok(Some(Self { condition: and }))
     }
 
     pub fn from_str(expression: &'a str) -> Result<Option<Self>> {
@@ -284,14 +276,6 @@ impl<'a> Filter<'a> {
                 let exist = index.exists_faceted_documents_ids(rtxn, field_id)?;
                 return Ok(exist);
             }
-            Condition::NotExists => {
-                let all_ids = index.documents_ids(rtxn)?;
-
-                let exist = Self::evaluate_operator(rtxn, index, field_id, &Condition::Exists)?;
-
-                let notexist = all_ids - exist;
-                return Ok(notexist);
-            }
             Condition::Equal(val) => {
                 let (_original_value, string_docids) = strings_db
                     .get(rtxn, &(field_id, &val.value().to_lowercase()))?
@@ -317,11 +301,10 @@ impl<'a> Filter<'a> {
                 return Ok(string_docids | number_docids);
             }
             Condition::NotEqual(val) => {
-                let all_numbers_ids = index.number_faceted_documents_ids(rtxn, field_id)?;
-                let all_strings_ids = index.string_faceted_documents_ids(rtxn, field_id)?;
                 let operator = Condition::Equal(val.clone());
                 let docids = Self::evaluate_operator(rtxn, index, field_id, &operator)?;
-                return Ok((all_numbers_ids | all_strings_ids) - docids);
+                let all_ids = index.documents_ids(rtxn)?;
+                return Ok(all_ids - docids);
             }
         };
 
@@ -367,6 +350,39 @@ impl<'a> Filter<'a> {
         filterable_fields: &HashSet<String>,
     ) -> Result<RoaringBitmap> {
         match &self.condition {
+            FilterCondition::Not(f) => {
+                let all_ids = index.documents_ids(rtxn)?;
+                let selected = Self::inner_evaluate(
+                    &(f.as_ref().clone()).into(),
+                    rtxn,
+                    index,
+                    filterable_fields,
+                )?;
+                return Ok(all_ids - selected);
+            }
+            FilterCondition::In { fid, els } => {
+                if crate::is_faceted(fid.value(), filterable_fields) {
+                    let field_ids_map = index.fields_ids_map(rtxn)?;
+
+                    if let Some(fid) = field_ids_map.id(fid.value()) {
+                        let mut bitmap = RoaringBitmap::new();
+
+                        for el in els {
+                            let op = Condition::Equal(el.clone());
+                            let el_bitmap = Self::evaluate_operator(rtxn, index, fid, &op)?;
+                            bitmap |= el_bitmap;
+                        }
+                        Ok(bitmap)
+                    } else {
+                        Ok(RoaringBitmap::new())
+                    }
+                } else {
+                    return Err(fid.as_external_error(FilterError::AttributeNotFilterable {
+                        attribute: fid.value(),
+                        filterable_fields: filterable_fields.clone(),
+                    }))?;
+                }
+            }
             FilterCondition::Condition { fid, op } => {
                 if crate::is_faceted(fid.value(), filterable_fields) {
                     let field_ids_map = index.fields_ids_map(rtxn)?;
@@ -397,38 +413,38 @@ impl<'a> Filter<'a> {
                     }
                 }
             }
-            FilterCondition::Or(lhs, rhs) => {
+            FilterCondition::Or(subfilters) => {
-                let lhs = Self::inner_evaluate(
+                let mut bitmap = RoaringBitmap::new();
-                    &(lhs.as_ref().clone()).into(),
+                for f in subfilters {
-                    rtxn,
+                    bitmap |=
-                    index,
+                        Self::inner_evaluate(&(f.clone()).into(), rtxn, index, filterable_fields)?;
-                    filterable_fields,
+                }
-                )?;
+                Ok(bitmap)
-                let rhs = Self::inner_evaluate(
+            }
-                    &(rhs.as_ref().clone()).into(),
+            FilterCondition::And(subfilters) => {
-                    rtxn,
+                let mut subfilters_iter = subfilters.iter();
-                    index,
+                if let Some(first_subfilter) = subfilters_iter.next() {
-                    filterable_fields,
+                    let mut bitmap = Self::inner_evaluate(
-                )?;
+                        &(first_subfilter.clone()).into(),
-                Ok(lhs | rhs)
+                        rtxn,
-            }
+                        index,
-            FilterCondition::And(lhs, rhs) => {
+                        filterable_fields,
-                let lhs = Self::inner_evaluate(
+                    )?;
-                    &(lhs.as_ref().clone()).into(),
+                    for f in subfilters_iter {
-                    rtxn,
+                        if bitmap.is_empty() {
-                    index,
+                            return Ok(bitmap);
-                    filterable_fields,
+                        }
-                )?;
+                        bitmap &= Self::inner_evaluate(
-                if lhs.is_empty() {
+                            &(f.clone()).into(),
-                    return Ok(lhs);
+                            rtxn,
+                            index,
+                            filterable_fields,
+                        )?;
+                    }
+                    Ok(bitmap)
+                } else {
+                    Ok(RoaringBitmap::new())
                 }
-                let rhs = Self::inner_evaluate(
-                    &(rhs.as_ref().clone()).into(),
-                    rtxn,
-                    index,
-                    filterable_fields,
-                )?;
-                Ok(lhs & rhs)
             }
             FilterCondition::GeoLowerThan { point, radius } => {
                 if filterable_fields.contains("_geo") {
@@ -467,17 +483,6 @@ impl<'a> Filter<'a> {
                     }))?;
                 }
             }
-            FilterCondition::GeoGreaterThan { point, radius } => {
-                let result = Self::inner_evaluate(
-                    &FilterCondition::GeoLowerThan { point: point.clone(), radius: radius.clone() }
-                        .into(),
-                    rtxn,
-                    index,
-                    filterable_fields,
-                )?;
-                let geo_faceted_doc_ids = index.geo_faceted_documents_ids(rtxn)?;
-                Ok(geo_faceted_doc_ids - result)
-            }
         }
     }
 }
@@ -732,12 +737,10 @@ mod tests {
             }
         }
 
-        let error = Filter::from_str(&filter_string).unwrap_err();
+        // Note: the filter used to be rejected for being too deep, but that is
-        assert!(
+        // no longer the case
-            error.to_string().starts_with("Too many filter conditions"),
+        let filter = Filter::from_str(&filter_string).unwrap();
-            "{}",
+        assert!(filter.is_some());
-            error.to_string()
-        );
     }
 
     #[test]

milli/tests/assets/test_set.ndjson

@@ -1,17 +1,17 @@
-{"id":"A","word_rank":0,"typo_rank":1,"proximity_rank":15,"attribute_rank":505,"exact_rank":5,"asc_desc_rank":0,"sort_by_rank":0,"geo_rank":43,"title":"hell o","description":"hell o is the fourteenth episode of the american television series glee performing songs with this word","tag":"blue","_geo": { "lat": 50.62984446145472, "lng": 3.085712705162039 },"":"", "opt1": [null]}
+{"id":"A","word_rank":0,"typo_rank":1,"proximity_rank":15,"attribute_rank":505,"exact_rank":5,"asc_desc_rank":0,"sort_by_rank":0,"geo_rank":43,"title":"hell o","description":"hell o is the fourteenth episode of the american television series glee performing songs with this word","tag":"blue","_geo": { "lat": 50.62984446145472, "lng": 3.085712705162039 },"":"", "opt1": [null], "tag_in": 1}
-{"id":"B","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":1,"sort_by_rank":2,"geo_rank":191,"title":"hello","description":"hello is a song recorded by english singer songwriter adele","tag":"red","_geo": { "lat": 50.63047567664291, "lng": 3.088852230809636 },"":"", "opt1": []}
+{"id":"B","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":1,"sort_by_rank":2,"geo_rank":191,"title":"hello","description":"hello is a song recorded by english singer songwriter adele","tag":"red","_geo": { "lat": 50.63047567664291, "lng": 3.088852230809636 },"":"", "opt1": [], "tag_in": 2}
-{"id":"C","word_rank":0,"typo_rank":1,"proximity_rank":8,"attribute_rank":336,"exact_rank":4,"asc_desc_rank":2,"sort_by_rank":0,"geo_rank":283,"title":"hell on earth","description":"hell on earth is the third studio album by american hip hop duo mobb deep","tag":"blue","_geo": { "lat": 50.6321800003937, "lng": 3.088331882262139 },"":"", "opt1": null}
+{"id":"C","word_rank":0,"typo_rank":1,"proximity_rank":8,"attribute_rank":336,"exact_rank":4,"asc_desc_rank":2,"sort_by_rank":0,"geo_rank":283,"title":"hell on earth","description":"hell on earth is the third studio album by american hip hop duo mobb deep","tag":"blue","_geo": { "lat": 50.6321800003937, "lng": 3.088331882262139 },"":"", "opt1": null, "tag_in": 3}
-{"id":"D","word_rank":0,"typo_rank":1,"proximity_rank":10,"attribute_rank":757,"exact_rank":4,"asc_desc_rank":3,"sort_by_rank":2,"geo_rank":1381,"title":"hell on wheels tv series","description":"the construction of the first transcontinental railroad across the united states in the world","tag":"red","_geo": { "lat": 50.63728851135729, "lng": 3.0703951595971626 },"":"", "opt1": 4}
+{"id":"D","word_rank":0,"typo_rank":1,"proximity_rank":10,"attribute_rank":757,"exact_rank":4,"asc_desc_rank":3,"sort_by_rank":2,"geo_rank":1381,"title":"hell on wheels tv series","description":"the construction of the first transcontinental railroad across the united states in the world","tag":"red","_geo": { "lat": 50.63728851135729, "lng": 3.0703951595971626 },"":"", "opt1": 4, "tag_in": "four"}
-{"id":"E","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":1979,"title":"hello kitty","description":"also known by her full name kitty white is a fictional character produced by the japanese company sanrio","tag":"green","_geo": { "lat": 50.64264610511925, "lng": 3.0665099941857634 },"":"", "opt1": "E"}
+{"id":"E","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":1979,"title":"hello kitty","description":"also known by her full name kitty white is a fictional character produced by the japanese company sanrio","tag":"green","_geo": { "lat": 50.64264610511925, "lng": 3.0665099941857634 },"":"", "opt1": "E", "tag_in": "five"}
-{"id":"F","word_rank":2,"typo_rank":1,"proximity_rank":0,"attribute_rank":1017,"exact_rank":5,"asc_desc_rank":5,"sort_by_rank":0,"geo_rank":65022,"title":"laptop orchestra","description":"a laptop orchestra lork or lo is a chamber music ensemble consisting primarily of laptops like helo huddersfield experimental laptop orchestra","tag":"blue","_geo": { "lat": 51.05028653642387, "lng": 3.7301072771642096 },"":"", "opt1": ["F"]}
+{"id":"F","word_rank":2,"typo_rank":1,"proximity_rank":0,"attribute_rank":1017,"exact_rank":5,"asc_desc_rank":5,"sort_by_rank":0,"geo_rank":65022,"title":"laptop orchestra","description":"a laptop orchestra lork or lo is a chamber music ensemble consisting primarily of laptops like helo huddersfield experimental laptop orchestra","tag":"blue","_geo": { "lat": 51.05028653642387, "lng": 3.7301072771642096 },"":"", "opt1": ["F"], "tag_in": null}
 {"id":"G","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":5,"sort_by_rank":2,"geo_rank":34692,"title":"hello world film","description":"hello world is a 2019 japanese animated sci fi romantic drama film directed by tomohiko ito and produced by graphinica","tag":"red","_geo": { "lat": 50.78776041427129, "lng": 2.661201766290338 },"":"", "opt1": [7]}
-{"id":"H","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":202182,"title":"world hello day","description":"holiday observed on november 21 to express that conflicts should be resolved through communication rather than the use of force","tag":"green","_geo": { "lat": 48.875617484531965, "lng": 2.346747821504194 },"":"", "opt1": ["H", 8]}
+{"id":"H","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":202182,"title":"world hello day","description":"holiday observed on november 21 to express that conflicts should be resolved through communication rather than the use of force","tag":"green","_geo": { "lat": 48.875617484531965, "lng": 2.346747821504194 },"":"", "opt1": ["H", 8], "tag_in": 8}
-{"id":"I","word_rank":0,"typo_rank":0,"proximity_rank":8,"attribute_rank":338,"exact_rank":3,"asc_desc_rank":3,"sort_by_rank":0,"geo_rank":740667,"title":"hello world song","description":"hello world is a song written by tom douglas tony lane and david lee and recorded by american country music group lady antebellum","tag":"blue","_geo": { "lat": 43.973998070351065, "lng": 3.4661837318345032 },"":""}
+{"id":"I","word_rank":0,"typo_rank":0,"proximity_rank":8,"attribute_rank":338,"exact_rank":3,"asc_desc_rank":3,"sort_by_rank":0,"geo_rank":740667,"title":"hello world song","description":"hello world is a song written by tom douglas tony lane and david lee and recorded by american country music group lady antebellum","tag":"blue","_geo": { "lat": 43.973998070351065, "lng": 3.4661837318345032 },"":"", "tag_in": "nine"}
-{"id":"J","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":1,"exact_rank":3,"asc_desc_rank":2,"sort_by_rank":1,"geo_rank":739020,"title":"hello cruel world","description":"hello cruel world is an album by new zealand band tall dwarfs","tag":"green","_geo": { "lat": 43.98920130353838, "lng": 3.480519311627928 },"":"", "opt1": {}}
+{"id":"J","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":1,"exact_rank":3,"asc_desc_rank":2,"sort_by_rank":1,"geo_rank":739020,"title":"hello cruel world","description":"hello cruel world is an album by new zealand band tall dwarfs","tag":"green","_geo": { "lat": 43.98920130353838, "lng": 3.480519311627928 },"":"", "opt1": {}, "tag_in": 10}
-{"id":"K","word_rank":0,"typo_rank":2,"proximity_rank":9,"attribute_rank":670,"exact_rank":5,"asc_desc_rank":1,"sort_by_rank":2,"geo_rank":738830,"title":"hallo creation system","description":"in few word hallo was a construction toy created by the american company mattel to engage girls in construction play","tag":"red","_geo": { "lat": 43.99155030238669, "lng": 3.503453528249425 },"":"", "opt1": [{"opt2": 11}] }
+{"id":"K","word_rank":0,"typo_rank":2,"proximity_rank":9,"attribute_rank":670,"exact_rank":5,"asc_desc_rank":1,"sort_by_rank":2,"geo_rank":738830,"title":"hallo creation system","description":"in few word hallo was a construction toy created by the american company mattel to engage girls in construction play","tag":"red","_geo": { "lat": 43.99155030238669, "lng": 3.503453528249425 },"":"", "opt1": [{"opt2": 11}] , "tag_in": "eleven"}
-{"id":"L","word_rank":0,"typo_rank":0,"proximity_rank":2,"attribute_rank":250,"exact_rank":4,"asc_desc_rank":0,"sort_by_rank":0,"geo_rank":737861,"title":"good morning world","description":"good morning world is an american sitcom broadcast on cbs tv during the 1967 1968 season","tag":"blue","_geo": { "lat": 44.000507750283695, "lng": 3.5116812040621572 },"":"", "opt1": {"opt2": [12]}}
+{"id":"L","word_rank":0,"typo_rank":0,"proximity_rank":2,"attribute_rank":250,"exact_rank":4,"asc_desc_rank":0,"sort_by_rank":0,"geo_rank":737861,"title":"good morning world","description":"good morning world is an american sitcom broadcast on cbs tv during the 1967 1968 season","tag":"blue","_geo": { "lat": 44.000507750283695, "lng": 3.5116812040621572 },"":"", "opt1": {"opt2": [12]}, "tag_in": 12}
 {"id":"M","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":0,"asc_desc_rank":0,"sort_by_rank":2,"geo_rank":739203,"title":"hello world america","description":"a perfect match for a perfect engine using the query hello world america","tag":"red","_geo": { "lat": 43.99150729038736, "lng": 3.606143957295055 },"":"", "opt1": [13, [{"opt2": null}]]}
-{"id":"N","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":1,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":9499586,"title":"hello world america unleashed","description":"a very good match for a very good engine using the query hello world america","tag":"green","_geo": { "lat": 35.511540843367115, "lng": 138.764368875787 },"":"", "opt1": {"a": 1, "opt2": {"opt3": 14}} }
+{"id":"N","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":1,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":9499586,"title":"hello world america unleashed","description":"a very good match for a very good engine using the query hello world america","tag":"green","_geo": { "lat": 35.511540843367115, "lng": 138.764368875787 },"":"", "opt1": {"a": 1, "opt2": {"opt3": 14}}}
-{"id":"O","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":10,"exact_rank":0,"asc_desc_rank":6,"sort_by_rank":0,"geo_rank":9425163,"title":"a perfect match for a perfect engine using the query hello world america","description":"hello world america","tag":"blue","_geo": { "lat": 35.00536702277189, "lng": 135.76118763940391 },"":"", "opt1": [[[[]]]]  }
+{"id":"O","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":10,"exact_rank":0,"asc_desc_rank":6,"sort_by_rank":0,"geo_rank":9425163,"title":"a perfect match for a perfect engine using the query hello world america","description":"hello world america","tag":"blue","_geo": { "lat": 35.00536702277189, "lng": 135.76118763940391 },"":"", "opt1": [[[[]]]]}
 {"id":"P","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":12,"exact_rank":1,"asc_desc_rank":3,"sort_by_rank":2,"geo_rank":9422437,"title":"a very good match for a very good engine using the query hello world america","description":"hello world america unleashed","tag":"red","_geo": { "lat": 35.06462306367058, "lng": 135.8338440354251 },"":"", "opt1.opt2": 16}
 {"id":"Q","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":2,"sort_by_rank":1,"geo_rank":9339230,"title":"hello world","description":"a hello world program generally is a computer program that outputs or displays the message hello world","tag":"green","_geo": { "lat": 34.39548365683149, "lng": 132.4535960928883 },"":""}

milli/tests/search/filters.rs

@@ -85,4 +85,6 @@ test_filter!(exists_filter_1_not, vec![Right("opt1 NOT EXISTS")]);
 test_filter!(exists_filter_1_not_alt, vec![Right("NOT opt1 EXISTS")]);
 test_filter!(exists_filter_1_double_not, vec![Right("NOT opt1 NOT EXISTS")]);
 
-test_filter!(exists_filter_2, vec![Right("opt1.opt2 EXISTS")]);
+test_filter!(in_filter, vec![Right("tag_in IN[1, 2, 3, four, five]")]);
+test_filter!(not_in_filter, vec![Right("tag_in NOT IN[1, 2, 3, four, five]")]);
+test_filter!(not_not_in_filter, vec![Right("NOT tag_in NOT IN[1, 2, 3, four, five]")]);

milli/tests/search/mod.rs

@@ -44,7 +44,8 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
         S("asc_desc_rank"),
         S("_geo"),
         S("opt1"),
-        S("opt1.opt2")
+        S("opt1.opt2"),
+        S("tag_in")
     });
     builder.set_sortable_fields(hashset! {
         S("tag"),
@@ -205,6 +206,15 @@ fn execute_filter(filter: &str, document: &TestDocument) -> Option<String> {
         } else if let Some(opt1) = &document.opt1 {
             id = contains_key_rec(opt1, "opt2").then(|| document.id.clone());
         }
+    } else if matches!(
+        filter,
+        "tag_in IN[1, 2, 3, four, five]" | "NOT tag_in NOT IN[1, 2, 3, four, five]"
+    ) {
+        id = matches!(document.id.as_str(), "A" | "B" | "C" | "D" | "E")
+            .then(|| document.id.clone());
+    } else if matches!(filter, "tag_in NOT IN[1, 2, 3, four, five]") {
+        id = (!matches!(document.id.as_str(), "A" | "B" | "C" | "D" | "E"))
+            .then(|| document.id.clone());
     }
     id
 }