From 7c0cd7172d02664af8f5f59e2520741185d5dca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 8 Mar 2023 16:57:42 +0100 Subject: [PATCH] Introduce the NULL and NOT value NULL operator --- filter-parser/src/condition.rs | 16 ++++++++++++++++ filter-parser/src/lib.rs | 20 ++++++++++++++++---- milli/src/index.rs | 12 ++++++++++++ milli/src/search/facet/filter.rs | 4 ++++ 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/filter-parser/src/condition.rs b/filter-parser/src/condition.rs index 735ffec0e..834fac8b8 100644 --- a/filter-parser/src/condition.rs +++ b/filter-parser/src/condition.rs @@ -20,6 +20,7 @@ pub enum Condition<'a> { GreaterThanOrEqual(Token<'a>), Equal(Token<'a>), NotEqual(Token<'a>), + Null, Exists, LowerThan(Token<'a>), LowerThanOrEqual(Token<'a>), @@ -44,6 +45,21 @@ pub fn parse_condition(input: Span) -> IResult { Ok((input, condition)) } +/// null = value "NULL" +pub fn parse_null(input: Span) -> IResult { + let (input, key) = terminated(parse_value, tag("NULL"))(input)?; + + Ok((input, FilterCondition::Condition { fid: key, op: Null })) +} + +/// null = value "NOT" WS+ "NULL" +pub fn parse_not_null(input: Span) -> IResult { + let (input, key) = parse_value(input)?; + + let (input, _) = tuple((tag("NOT"), multispace1, tag("NULL")))(input)?; + Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Null })))) +} + /// exist = value "EXISTS" pub fn parse_exists(input: Span) -> IResult { let (input, key) = terminated(parse_value, tag("EXISTS"))(input)?; diff --git a/filter-parser/src/lib.rs b/filter-parser/src/lib.rs index 8e21ff6be..36657587f 100644 --- a/filter-parser/src/lib.rs +++ b/filter-parser/src/lib.rs @@ -47,7 +47,7 @@ mod value; use std::fmt::Debug; pub use condition::{parse_condition, parse_to, Condition}; -use condition::{parse_exists, parse_not_exists}; +use condition::{parse_exists, parse_not_exists, parse_not_null, parse_null}; use error::{cut_with_err, ExpectedValueKind, NomErrorExt}; pub use error::{Error, ErrorKind}; use nom::branch::alt; @@ -414,6 +414,8 @@ fn parse_primary(input: Span, depth: usize) -> IResult { parse_in, parse_not_in, parse_condition, + parse_null, + parse_not_null, parse_exists, parse_not_exists, parse_to, @@ -496,14 +498,23 @@ pub mod tests { insta::assert_display_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}"); insta::assert_display_snapshot!(p("subscribers 100 TO 1000"), @"{subscribers} {100} TO {1000}"); - // Test NOT + EXISTS - insta::assert_display_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS"); + // Test NOT insta::assert_display_snapshot!(p("NOT subscribers < 1000"), @"NOT ({subscribers} < {1000})"); + insta::assert_display_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})"); + + // Test NULL + NOT NULL + insta::assert_display_snapshot!(p("subscribers NULL"), @"{subscribers} NULL"); + insta::assert_display_snapshot!(p("NOT subscribers NULL"), @"NOT ({subscribers} NULL)"); + insta::assert_display_snapshot!(p("subscribers NOT NULL"), @"NOT ({subscribers} NULL)"); + insta::assert_display_snapshot!(p("NOT subscribers NOT NULL"), @"{subscribers} NULL"); + insta::assert_display_snapshot!(p("subscribers NOT NULL"), @"NOT ({subscribers} NULL)"); + + // Test EXISTS + NOT EXITS + insta::assert_display_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS"); insta::assert_display_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)"); insta::assert_display_snapshot!(p("subscribers NOT EXISTS"), @"NOT ({subscribers} EXISTS)"); insta::assert_display_snapshot!(p("NOT subscribers NOT EXISTS"), @"{subscribers} EXISTS"); insta::assert_display_snapshot!(p("subscribers NOT EXISTS"), @"NOT ({subscribers} EXISTS)"); - insta::assert_display_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})"); // Test nested NOT insta::assert_display_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}"); @@ -800,6 +811,7 @@ impl<'a> std::fmt::Display for Condition<'a> { Condition::GreaterThanOrEqual(token) => write!(f, ">= {token}"), Condition::Equal(token) => write!(f, "= {token}"), Condition::NotEqual(token) => write!(f, "!= {token}"), + Condition::Null => write!(f, "NULL"), Condition::Exists => write!(f, "EXISTS"), Condition::LowerThan(token) => write!(f, "< {token}"), Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"), diff --git a/milli/src/index.rs b/milli/src/index.rs index ae7bd211e..3316028df 100644 --- a/milli/src/index.rs +++ b/milli/src/index.rs @@ -839,6 +839,18 @@ impl Index { } } + /// Retrieve all the documents which contain this field id set as null + pub fn null_faceted_documents_ids( + &self, + rtxn: &RoTxn, + field_id: FieldId, + ) -> heed::Result { + match self.facet_id_is_null_docids.get(rtxn, &BEU16::new(field_id))? { + Some(docids) => Ok(docids), + None => Ok(RoaringBitmap::new()), + } + } + /// Retrieve all the documents which contain this field id pub fn exists_faceted_documents_ids( &self, diff --git a/milli/src/search/facet/filter.rs b/milli/src/search/facet/filter.rs index a4ac53950..df42725c5 100644 --- a/milli/src/search/facet/filter.rs +++ b/milli/src/search/facet/filter.rs @@ -219,6 +219,10 @@ impl<'a> Filter<'a> { Condition::Between { from, to } => { (Included(from.parse_finite_float()?), Included(to.parse_finite_float()?)) } + Condition::Null => { + let is_null = index.null_faceted_documents_ids(rtxn, field_id)?; + return Ok(is_null); + } Condition::Exists => { let exist = index.exists_faceted_documents_ids(rtxn, field_id)?; return Ok(exist);