diff --git a/http-ui/public/script.js b/http-ui/public/script.js index b621cd453..e4de86672 100644 --- a/http-ui/public/script.js +++ b/http-ui/public/script.js @@ -60,7 +60,13 @@ $('#query, #filters').on('input', function () { const content = document.createElement('div'); content.classList.add("content"); - content.innerHTML = element[prop]; + + // Stringify Objects and Arrays to avoid [Object object] + if (typeof element[prop] === 'object' && element[prop] !== null) { + content.innerHTML = JSON.stringify(element[prop]); + } else { + content.innerHTML = element[prop]; + } field.appendChild(attribute); field.appendChild(content); diff --git a/http-ui/src/main.rs b/http-ui/src/main.rs index 108ec0549..5dbb0c326 100644 --- a/http-ui/src/main.rs +++ b/http-ui/src/main.rs @@ -695,6 +695,7 @@ async fn main() -> anyhow::Result<()> { struct QueryBody { query: Option, filters: Option, + sort: Option, facet_filters: Option, String>>>, facet_distribution: Option, limit: Option, @@ -754,6 +755,10 @@ async fn main() -> anyhow::Result<()> { search.limit(limit); } + if let Some(sort) = query.sort { + search.sort_criteria(vec![sort.parse().unwrap()]); + } + let SearchResult { matching_words, candidates, documents_ids } = search.execute().unwrap(); diff --git a/milli/Cargo.toml b/milli/Cargo.toml index 98eac15a9..a8f80d437 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -16,6 +16,7 @@ flate2 = "1.0.20" fst = "0.4.5" fxhash = "0.2.1" grenad = { version = "0.3.1", default-features = false, features = ["tempfile"] } +geoutils = "0.4.1" heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] } human_format = "1.0.3" levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] } @@ -27,6 +28,7 @@ once_cell = "1.5.2" ordered-float = "2.1.1" rayon = "1.5.0" roaring = "0.6.6" +rstar = { version = "0.9.1", features = ["serde"] } serde = { version = "1.0.123", features = ["derive"] } serde_json = { version = "1.0.62", features = ["preserve_order"] } slice-group-by = "0.2.6" diff --git a/milli/src/criterion.rs b/milli/src/criterion.rs index d91d4a7e1..24879cdd4 100644 --- a/milli/src/criterion.rs +++ b/milli/src/criterion.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use serde::{Deserialize, Serialize}; -use crate::error::{Error, UserError}; +use crate::error::{is_reserved_keyword, Error, UserError}; #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub enum Criterion { @@ -50,32 +50,101 @@ impl FromStr for Criterion { "sort" => Ok(Criterion::Sort), "exactness" => Ok(Criterion::Exactness), text => match AscDesc::from_str(text) { - Ok(AscDesc::Asc(field)) => Ok(Criterion::Asc(field)), - Ok(AscDesc::Desc(field)) => Ok(Criterion::Desc(field)), + Ok(AscDesc::Asc(Member::Field(field))) => Ok(Criterion::Asc(field)), + Ok(AscDesc::Desc(Member::Field(field))) => Ok(Criterion::Desc(field)), + Ok(AscDesc::Asc(Member::Geo(_))) | Ok(AscDesc::Desc(Member::Geo(_))) => { + Err(UserError::InvalidRankingRuleName { name: text.to_string() })? + } Err(UserError::InvalidAscDescSyntax { name }) => { - Err(UserError::InvalidCriterionName { name }.into()) + Err(UserError::InvalidRankingRuleName { name }.into()) } Err(error) => { - Err(UserError::InvalidCriterionName { name: error.to_string() }.into()) + Err(UserError::InvalidRankingRuleName { name: error.to_string() }.into()) } }, } } } -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub enum Member { + Field(String), + Geo([f64; 2]), +} + +impl FromStr for Member { + type Err = UserError; + + fn from_str(text: &str) -> Result { + match text.strip_prefix("_geoPoint(").and_then(|text| text.strip_suffix(")")) { + Some(point) => { + let (lat, long) = point + .split_once(',') + .ok_or_else(|| UserError::InvalidRankingRuleName { name: text.to_string() }) + .and_then(|(lat, long)| { + lat.trim() + .parse() + .and_then(|lat| long.trim().parse().map(|long| (lat, long))) + .map_err(|_| UserError::InvalidRankingRuleName { + name: text.to_string(), + }) + })?; + Ok(Member::Geo([lat, long])) + } + None => { + if is_reserved_keyword(text) { + return Err(UserError::InvalidReservedRankingRuleName { + name: text.to_string(), + })?; + } + Ok(Member::Field(text.to_string())) + } + } + } +} + +impl fmt::Display for Member { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Member::Field(name) => f.write_str(name), + Member::Geo([lat, lng]) => write!(f, "_geoPoint({}, {})", lat, lng), + } + } +} + +impl Member { + pub fn field(&self) -> Option<&str> { + match self { + Member::Field(field) => Some(field), + Member::Geo(_) => None, + } + } + + pub fn geo_point(&self) -> Option<&[f64; 2]> { + match self { + Member::Geo(point) => Some(point), + Member::Field(_) => None, + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] pub enum AscDesc { - Asc(String), - Desc(String), + Asc(Member), + Desc(Member), } impl AscDesc { - pub fn field(&self) -> &str { + pub fn member(&self) -> &Member { match self { - AscDesc::Asc(field) => field, - AscDesc::Desc(field) => field, + AscDesc::Asc(member) => member, + AscDesc::Desc(member) => member, } } + + pub fn field(&self) -> Option<&str> { + self.member().field() + } } impl FromStr for AscDesc { @@ -85,9 +154,9 @@ impl FromStr for AscDesc { /// string and let the caller create his own error fn from_str(text: &str) -> Result { match text.rsplit_once(':') { - Some((field_name, "asc")) => Ok(AscDesc::Asc(field_name.to_string())), - Some((field_name, "desc")) => Ok(AscDesc::Desc(field_name.to_string())), - _ => Err(UserError::InvalidAscDescSyntax { name: text.to_string() }), + Some((left, "asc")) => Ok(AscDesc::Asc(left.parse()?)), + Some((left, "desc")) => Ok(AscDesc::Desc(left.parse()?)), + _ => Err(UserError::InvalidRankingRuleName { name: text.to_string() }), } } } @@ -119,3 +188,63 @@ impl fmt::Display for Criterion { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_asc_desc() { + use big_s::S; + use AscDesc::*; + use Member::*; + + let valid_req = [ + ("truc:asc", Asc(Field(S("truc")))), + ("bidule:desc", Desc(Field(S("bidule")))), + ("a-b:desc", Desc(Field(S("a-b")))), + ("a:b:desc", Desc(Field(S("a:b")))), + ("a12:asc", Asc(Field(S("a12")))), + ("42:asc", Asc(Field(S("42")))), + ("_geoPoint(42, 59):asc", Asc(Geo([42., 59.]))), + ("_geoPoint(42.459, 59):desc", Desc(Geo([42.459, 59.]))), + ("_geoPoint(42, 59.895):desc", Desc(Geo([42., 59.895]))), + ("_geoPoint(42, 59.895):desc", Desc(Geo([42., 59.895]))), + ("_geoPoint(42.0002, 59.895):desc", Desc(Geo([42.0002, 59.895]))), + ("_geoPoint(42., 59.):desc", Desc(Geo([42., 59.]))), + ("truc(12, 13):desc", Desc(Field(S("truc(12, 13)")))), + ]; + + for (req, expected) in valid_req { + let res = req.parse(); + assert!(res.is_ok(), "Failed to parse `{}`, was expecting `{:?}`", req, expected); + assert_eq!(expected, res.unwrap()); + } + + let invalid_req = [ + "truc:machin", + "truc:deesc", + "truc:asc:deesc", + "42desc", + "_geoPoint:asc", + "_geoDistance:asc", + "_geoPoint(42.12 , 59.598)", + "_geoPoint(42.12 , 59.598):deesc", + "_geoPoint(42.12 , 59.598):machin", + "_geoPoint(42.12 , 59.598):asc:aasc", + "_geoPoint(42,12 , 59,598):desc", + "_geoPoint(35, 85, 75):asc", + "_geoPoint(18):asc", + ]; + + for req in invalid_req { + let res = req.parse::(); + assert!( + res.is_err(), + "Should no be able to parse `{}`, was expecting an error but instead got: `{:?}`", + req, + res, + ); + } + } +} diff --git a/milli/src/error.rs b/milli/src/error.rs index 3f473a673..e6bd3fd62 100644 --- a/milli/src/error.rs +++ b/milli/src/error.rs @@ -12,6 +12,10 @@ use crate::{DocumentId, FieldId}; pub type Object = Map; +pub fn is_reserved_keyword(keyword: &str) -> bool { + ["_geo", "_geoDistance", "_geoPoint", "_geoRadius"].contains(&keyword) +} + #[derive(Debug)] pub enum Error { InternalError(InternalError), @@ -54,12 +58,14 @@ pub enum UserError { Csv(csv::Error), DocumentLimitReached, InvalidAscDescSyntax { name: String }, - InvalidCriterionName { name: String }, InvalidDocumentId { document_id: Value }, InvalidFacetsDistribution { invalid_facets_name: HashSet }, InvalidFilter(pest::error::Error), InvalidFilterAttribute(pest::error::Error), InvalidSortName { name: String }, + InvalidGeoField { document_id: Value, object: Value }, + InvalidRankingRuleName { name: String }, + InvalidReservedRankingRuleName { name: String }, InvalidSortableAttribute { field: String, valid_fields: HashSet }, SortRankingRuleMissing, InvalidStoreFile, @@ -221,7 +227,15 @@ impl fmt::Display for UserError { Self::InvalidAscDescSyntax { name } => { write!(f, "invalid asc/desc syntax for {}", name) } - Self::InvalidCriterionName { name } => write!(f, "invalid criterion {}", name), + Self::InvalidGeoField { document_id, object } => write!( + f, + "the document with the id: {} contains an invalid _geo field: {}", + document_id, object + ), + Self::InvalidRankingRuleName { name } => write!(f, "invalid criterion {}", name), + Self::InvalidReservedRankingRuleName { name } => { + write!(f, "{} is a reserved keyword and thus can't be used as a ranking rule", name) + } Self::InvalidDocumentId { document_id } => { let json = serde_json::to_string(document_id).unwrap(); write!( diff --git a/milli/src/index.rs b/milli/src/index.rs index f3a2a3e05..f7603148d 100644 --- a/milli/src/index.rs +++ b/milli/src/index.rs @@ -8,6 +8,7 @@ use heed::flags::Flags; use heed::types::*; use heed::{Database, PolyDatabase, RoTxn, RwTxn}; use roaring::RoaringBitmap; +use rstar::RTree; use crate::error::{InternalError, UserError}; use crate::fields_ids_map::FieldsIdsMap; @@ -18,8 +19,8 @@ use crate::heed_codec::facet::{ use crate::{ default_criteria, BEU32StrCodec, BoRoaringBitmapCodec, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds, FacetDistribution, FieldDistribution, FieldId, - FieldIdWordCountCodec, ObkvCodec, Result, RoaringBitmapCodec, RoaringBitmapLenCodec, Search, - StrLevelPositionCodec, StrStrU8Codec, BEU32, + FieldIdWordCountCodec, GeoPoint, ObkvCodec, Result, RoaringBitmapCodec, RoaringBitmapLenCodec, + Search, StrLevelPositionCodec, StrStrU8Codec, BEU32, }; pub mod main_key { @@ -31,6 +32,8 @@ pub mod main_key { pub const SORTABLE_FIELDS_KEY: &str = "sortable-fields"; pub const FIELD_DISTRIBUTION_KEY: &str = "fields-distribution"; pub const FIELDS_IDS_MAP_KEY: &str = "fields-ids-map"; + pub const GEO_FACETED_DOCUMENTS_IDS_KEY: &str = "geo-faceted-documents-ids"; + pub const GEO_RTREE_KEY: &str = "geo-rtree"; pub const HARD_EXTERNAL_DOCUMENTS_IDS_KEY: &str = "hard-external-documents-ids"; pub const NUMBER_FACETED_DOCUMENTS_IDS_PREFIX: &str = "number-faceted-documents-ids"; pub const PRIMARY_KEY_KEY: &str = "primary-key"; @@ -294,6 +297,64 @@ impl Index { .unwrap_or_default()) } + /* geo rtree */ + + /// Writes the provided `rtree` which associates coordinates to documents ids. + pub(crate) fn put_geo_rtree( + &self, + wtxn: &mut RwTxn, + rtree: &RTree, + ) -> heed::Result<()> { + self.main.put::<_, Str, SerdeBincode>>(wtxn, main_key::GEO_RTREE_KEY, rtree) + } + + /// Delete the `rtree` which associates coordinates to documents ids. + pub(crate) fn delete_geo_rtree(&self, wtxn: &mut RwTxn) -> heed::Result { + self.main.delete::<_, Str>(wtxn, main_key::GEO_RTREE_KEY) + } + + /// Returns the `rtree` which associates coordinates to documents ids. + pub fn geo_rtree<'t>(&self, rtxn: &'t RoTxn) -> Result>> { + match self + .main + .get::<_, Str, SerdeBincode>>(rtxn, main_key::GEO_RTREE_KEY)? + { + Some(rtree) => Ok(Some(rtree)), + None => Ok(None), + } + } + + /* geo faceted */ + + /// Writes the documents ids that are faceted with a _geo field. + pub(crate) fn put_geo_faceted_documents_ids( + &self, + wtxn: &mut RwTxn, + docids: &RoaringBitmap, + ) -> heed::Result<()> { + self.main.put::<_, Str, RoaringBitmapCodec>( + wtxn, + main_key::GEO_FACETED_DOCUMENTS_IDS_KEY, + docids, + ) + } + + /// Delete the documents ids that are faceted with a _geo field. + pub(crate) fn delete_geo_faceted_documents_ids(&self, wtxn: &mut RwTxn) -> heed::Result { + self.main.delete::<_, Str>(wtxn, main_key::GEO_FACETED_DOCUMENTS_IDS_KEY) + } + + /// Retrieve all the documents ids that are faceted with a _geo field. + pub fn geo_faceted_documents_ids(&self, rtxn: &RoTxn) -> heed::Result { + match self + .main + .get::<_, Str, RoaringBitmapCodec>(rtxn, main_key::GEO_FACETED_DOCUMENTS_IDS_KEY)? + { + Some(docids) => Ok(docids), + None => Ok(RoaringBitmap::new()), + } + } + /* field distribution */ /// Writes the field distribution which associates every field name with diff --git a/milli/src/lib.rs b/milli/src/lib.rs index af811fe08..7c9f56665 100644 --- a/milli/src/lib.rs +++ b/milli/src/lib.rs @@ -21,7 +21,7 @@ use fxhash::{FxHasher32, FxHasher64}; pub use grenad::CompressionType; use serde_json::{Map, Value}; -pub use self::criterion::{default_criteria, AscDesc, Criterion}; +pub use self::criterion::{default_criteria, AscDesc, Criterion, Member}; pub use self::error::{ Error, FieldIdMapMissingEntry, InternalError, SerializationError, UserError, }; @@ -51,6 +51,7 @@ pub type DocumentId = u32; pub type FieldId = u16; pub type Position = u32; pub type FieldDistribution = BTreeMap; +pub type GeoPoint = rstar::primitives::GeomWithData<[f64; 2], DocumentId>; /// Transform a raw obkv store into a JSON Object. pub fn obkv_to_json( @@ -141,6 +142,15 @@ where Some((head, tail)) } +/// Return the distance between two points in meters. Each points are composed of two f64, +/// one latitude and one longitude. +pub fn distance_between_two_points(a: &[f64; 2], b: &[f64; 2]) -> f64 { + let a = geoutils::Location::new(a[0], a[1]); + let b = geoutils::Location::new(b[0], b[1]); + + a.haversine_distance_to(&b).meters() +} + #[cfg(test)] mod tests { use serde_json::json; diff --git a/milli/src/search/criteria/geo.rs b/milli/src/search/criteria/geo.rs new file mode 100644 index 000000000..de6de8912 --- /dev/null +++ b/milli/src/search/criteria/geo.rs @@ -0,0 +1,150 @@ +use std::iter; + +use roaring::RoaringBitmap; +use rstar::RTree; + +use super::{Criterion, CriterionParameters, CriterionResult}; +use crate::search::criteria::{resolve_query_tree, CriteriaBuilder}; +use crate::{GeoPoint, Index, Result}; + +pub struct Geo<'t> { + index: &'t Index, + rtxn: &'t heed::RoTxn<'t>, + ascending: bool, + parent: Box, + candidates: Box>, + allowed_candidates: RoaringBitmap, + bucket_candidates: RoaringBitmap, + rtree: Option>, + point: [f64; 2], +} + +impl<'t> Geo<'t> { + pub fn asc( + index: &'t Index, + rtxn: &'t heed::RoTxn<'t>, + parent: Box, + point: [f64; 2], + ) -> Result { + Self::new(index, rtxn, parent, point, true) + } + + pub fn desc( + index: &'t Index, + rtxn: &'t heed::RoTxn<'t>, + parent: Box, + point: [f64; 2], + ) -> Result { + Self::new(index, rtxn, parent, point, false) + } + + fn new( + index: &'t Index, + rtxn: &'t heed::RoTxn<'t>, + parent: Box, + point: [f64; 2], + ascending: bool, + ) -> Result { + let candidates = Box::new(iter::empty()); + let allowed_candidates = index.geo_faceted_documents_ids(rtxn)?; + let bucket_candidates = RoaringBitmap::new(); + let rtree = index.geo_rtree(rtxn)?; + + Ok(Self { + index, + rtxn, + ascending, + parent, + candidates, + allowed_candidates, + bucket_candidates, + rtree, + point, + }) + } +} + +impl Criterion for Geo<'_> { + fn next(&mut self, params: &mut CriterionParameters) -> Result> { + let rtree = self.rtree.as_ref(); + + loop { + match self.candidates.next() { + Some(mut candidates) => { + candidates -= params.excluded_candidates; + self.allowed_candidates -= &candidates; + return Ok(Some(CriterionResult { + query_tree: None, + candidates: Some(candidates), + filtered_candidates: None, + bucket_candidates: Some(self.bucket_candidates.clone()), + })); + } + None => match self.parent.next(params)? { + Some(CriterionResult { + query_tree, + candidates, + filtered_candidates, + bucket_candidates, + }) => { + let mut candidates = match (&query_tree, candidates) { + (_, Some(candidates)) => candidates, + (Some(qt), None) => { + let context = CriteriaBuilder::new(&self.rtxn, &self.index)?; + resolve_query_tree(&context, qt, params.wdcache)? + } + (None, None) => self.index.documents_ids(self.rtxn)?, + }; + + if let Some(filtered_candidates) = filtered_candidates { + candidates &= filtered_candidates; + } + + match bucket_candidates { + Some(bucket_candidates) => self.bucket_candidates |= bucket_candidates, + None => self.bucket_candidates |= &candidates, + } + + if candidates.is_empty() { + continue; + } + self.allowed_candidates = &candidates - params.excluded_candidates; + self.candidates = match rtree { + Some(rtree) => geo_point( + rtree, + self.allowed_candidates.clone(), + self.point, + self.ascending, + ), + None => Box::new(std::iter::empty()), + }; + } + None => return Ok(None), + }, + } + } + } +} + +fn geo_point( + rtree: &RTree, + mut candidates: RoaringBitmap, + point: [f64; 2], + ascending: bool, +) -> Box> { + let mut results = Vec::new(); + for point in rtree.nearest_neighbor_iter(&point) { + if candidates.remove(point.data) { + results.push(std::iter::once(point.data).collect()); + if candidates.is_empty() { + break; + } + } + } + + if ascending { + Box::new(results.into_iter()) + } else { + Box::new(results.into_iter().rev()) + } +} diff --git a/milli/src/search/criteria/mod.rs b/milli/src/search/criteria/mod.rs index 2a883de67..c2de55de5 100644 --- a/milli/src/search/criteria/mod.rs +++ b/milli/src/search/criteria/mod.rs @@ -12,7 +12,8 @@ use self::r#final::Final; use self::typo::Typo; use self::words::Words; use super::query_tree::{Operation, PrimitiveQueryPart, Query, QueryKind}; -use crate::criterion::AscDesc as AscDescName; +use crate::criterion::{AscDesc as AscDescName, Member}; +use crate::search::criteria::geo::Geo; use crate::search::{word_derivations, WordDerivationsCache}; use crate::{DocumentId, FieldId, Index, Result, TreeLevel}; @@ -20,6 +21,7 @@ mod asc_desc; mod attribute; mod exactness; pub mod r#final; +mod geo; mod initial; mod proximity; mod typo; @@ -290,18 +292,30 @@ impl<'t> CriteriaBuilder<'t> { Some(ref sort_criteria) => { for asc_desc in sort_criteria { criterion = match asc_desc { - AscDescName::Asc(field) => Box::new(AscDesc::asc( + AscDescName::Asc(Member::Field(field)) => Box::new(AscDesc::asc( &self.index, &self.rtxn, criterion, field.to_string(), )?), - AscDescName::Desc(field) => Box::new(AscDesc::desc( + AscDescName::Desc(Member::Field(field)) => Box::new(AscDesc::desc( &self.index, &self.rtxn, criterion, field.to_string(), )?), + AscDescName::Asc(Member::Geo(point)) => Box::new(Geo::asc( + &self.index, + &self.rtxn, + criterion, + point.clone(), + )?), + AscDescName::Desc(Member::Geo(point)) => Box::new(Geo::desc( + &self.index, + &self.rtxn, + criterion, + point.clone(), + )?), }; } criterion diff --git a/milli/src/search/facet/filter_condition.rs b/milli/src/search/facet/filter_condition.rs index a92797e90..1e5bf9ad0 100644 --- a/milli/src/search/facet/filter_condition.rs +++ b/milli/src/search/facet/filter_condition.rs @@ -21,7 +21,9 @@ use crate::error::UserError; use crate::heed_codec::facet::{ FacetLevelValueF64Codec, FacetStringLevelZeroCodec, FacetStringLevelZeroValueCodec, }; -use crate::{CboRoaringBitmapCodec, FieldId, FieldsIdsMap, Index, Result}; +use crate::{ + distance_between_two_points, CboRoaringBitmapCodec, FieldId, FieldsIdsMap, Index, Result, +}; #[derive(Debug, Clone, PartialEq)] pub enum Operator { @@ -32,6 +34,8 @@ pub enum Operator { LowerThan(f64), LowerThanOrEqual(f64), Between(f64, f64), + GeoLowerThan([f64; 2], f64), + GeoGreaterThan([f64; 2], f64), } impl Operator { @@ -46,6 +50,8 @@ impl Operator { LowerThan(n) => (GreaterThanOrEqual(n), None), LowerThanOrEqual(n) => (GreaterThan(n), None), Between(n, m) => (LowerThan(n), Some(GreaterThan(m))), + GeoLowerThan(point, distance) => (GeoGreaterThan(point, distance), None), + GeoGreaterThan(point, distance) => (GeoLowerThan(point, distance), None), } } } @@ -131,6 +137,7 @@ impl FilterCondition { Rule::leq => Ok(Self::lower_than_or_equal(fim, ff, pair)?), Rule::less => Ok(Self::lower_than(fim, ff, pair)?), Rule::between => Ok(Self::between(fim, ff, pair)?), + Rule::geo_radius => Ok(Self::geo_radius(fim, ff, pair)?), Rule::not => Ok(Self::from_pairs(fim, ff, pair.into_inner())?.negate()), Rule::prgm => Self::from_pairs(fim, ff, pair.into_inner()), Rule::term => Self::from_pairs(fim, ff, pair.into_inner()), @@ -156,6 +163,65 @@ impl FilterCondition { } } + fn geo_radius( + fields_ids_map: &FieldsIdsMap, + filterable_fields: &HashSet, + item: Pair, + ) -> Result { + if !filterable_fields.contains("_geo") { + return Err(UserError::InvalidFilterAttribute(PestError::new_from_span( + ErrorVariant::CustomError { + message: format!( + "attribute `_geo` is not filterable, available filterable attributes are: {}", + filterable_fields.iter().join(", "), + ), + }, + item.as_span(), + )))?; + } + let mut items = item.into_inner(); + let fid = match fields_ids_map.id("_geo") { + Some(fid) => fid, + None => return Ok(Empty), + }; + let parameters_item = items.next().unwrap(); + // We don't need more than 3 parameters, but to handle errors correctly we are still going + // to extract the first 4 parameters + let param_span = parameters_item.as_span(); + let parameters = parameters_item + .into_inner() + .take(4) + .map(|param| (param.clone(), param.as_span())) + .map(|(param, span)| pest_parse(param).0.map(|arg| (arg, span))) + .collect::, _>>() + .map_err(UserError::InvalidFilter)?; + if parameters.len() != 3 { + return Err(UserError::InvalidFilter(PestError::new_from_span( + ErrorVariant::CustomError { + message: format!("The `_geoRadius` filter expect three arguments: `_geoRadius(latitude, longitude, radius)`"), + }, + // we want to point to the last parameters and if there was no parameters we + // point to the parenthesis + parameters.last().map(|param| param.1.clone()).unwrap_or(param_span), + )))?; + } + let (lat, lng, distance) = (¶meters[0], ¶meters[1], parameters[2].0); + if let Some(span) = (!(-181.0..181.).contains(&lat.0)) + .then(|| &lat.1) + .or((!(-181.0..181.).contains(&lng.0)).then(|| &lng.1)) + { + return Err(UserError::InvalidFilter(PestError::new_from_span( + ErrorVariant::CustomError { + message: format!( + "Latitude and longitude must be contained between -180 to 180 degrees." + ), + }, + span.clone(), + )))?; + } + Ok(Operator(fid, GeoLowerThan([lat.0, lng.0], distance))) + } + fn between( fields_ids_map: &FieldsIdsMap, filterable_fields: &HashSet, @@ -440,6 +506,34 @@ impl FilterCondition { LowerThan(val) => (Included(f64::MIN), Excluded(*val)), LowerThanOrEqual(val) => (Included(f64::MIN), Included(*val)), Between(left, right) => (Included(*left), Included(*right)), + GeoLowerThan(base_point, distance) => { + let rtree = match index.geo_rtree(rtxn)? { + Some(rtree) => rtree, + None => return Ok(RoaringBitmap::new()), + }; + + let result = rtree + .nearest_neighbor_iter(base_point) + .take_while(|point| { + distance_between_two_points(base_point, point.geom()) < *distance + }) + .map(|point| point.data) + .collect(); + + return Ok(result); + } + GeoGreaterThan(point, distance) => { + let result = Self::evaluate_operator( + rtxn, + index, + numbers_db, + strings_db, + field_id, + &GeoLowerThan(point.clone(), *distance), + )?; + let geo_faceted_doc_ids = index.geo_faceted_documents_ids(rtxn)?; + return Ok(geo_faceted_doc_ids - result); + } }; // Ask for the biggest value that can exist for this specific field, if it exists @@ -505,6 +599,19 @@ fn field_id( ) -> StdResult, PestError> { // lexing ensures that we at least have a key let key = items.next().unwrap(); + if key.as_rule() == Rule::reserved { + return Err(PestError::new_from_span( + ErrorVariant::CustomError { + message: format!( + "`{}` is a reserved keyword and therefore can't be used as a filter expression. \ + Available filterable attributes are: {}", + key.as_str(), + filterable_fields.iter().join(", "), + ), + }, + key.as_span(), + )); + } if !filterable_fields.contains(key.as_str()) { return Err(PestError::new_from_span( @@ -581,6 +688,13 @@ mod tests { let condition = FilterCondition::from_str(&rtxn, &index, "NOT channel = ponce").unwrap(); let expected = Operator(0, Operator::NotEqual(None, S("ponce"))); assert_eq!(condition, expected); + + let result = FilterCondition::from_str(&rtxn, &index, "_geo = France"); + assert!(result.is_err()); + let error = result.unwrap_err(); + assert!(error.to_string().contains( + "`_geo` is a reserved keyword and therefore can't be used as a filter expression." + )); } #[test] @@ -663,6 +777,92 @@ mod tests { assert_eq!(condition, expected); } + #[test] + fn geo_radius() { + let path = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(10 * 1024 * 1024); // 10 MB + let index = Index::new(options, &path).unwrap(); + + // Set the filterable fields to be the channel. + let mut wtxn = index.write_txn().unwrap(); + let mut builder = Settings::new(&mut wtxn, &index, 0); + builder.set_searchable_fields(vec![S("_geo"), S("price")]); // to keep the fields order + builder.set_filterable_fields(hashset! { S("_geo"), S("price") }); + builder.execute(|_, _| ()).unwrap(); + wtxn.commit().unwrap(); + + let rtxn = index.read_txn().unwrap(); + // basic test + let condition = + FilterCondition::from_str(&rtxn, &index, "_geoRadius(12, 13.0005, 2000)").unwrap(); + let expected = Operator(0, GeoLowerThan([12., 13.0005], 2000.)); + assert_eq!(condition, expected); + + // test the negation of the GeoLowerThan + let condition = + FilterCondition::from_str(&rtxn, &index, "NOT _geoRadius(50, 18, 2000.500)").unwrap(); + let expected = Operator(0, GeoGreaterThan([50., 18.], 2000.500)); + assert_eq!(condition, expected); + + // composition of multiple operations + let condition = FilterCondition::from_str( + &rtxn, + &index, + "(NOT _geoRadius(1, 2, 300) AND _geoRadius(1.001, 2.002, 1000.300)) OR price <= 10", + ) + .unwrap(); + let expected = Or( + Box::new(And( + Box::new(Operator(0, GeoGreaterThan([1., 2.], 300.))), + Box::new(Operator(0, GeoLowerThan([1.001, 2.002], 1000.300))), + )), + Box::new(Operator(1, LowerThanOrEqual(10.))), + ); + assert_eq!(condition, expected); + + // georadius don't have any parameters + let result = FilterCondition::from_str(&rtxn, &index, "_geoRadius"); + assert!(result.is_err()); + let error = result.unwrap_err(); + assert!(error.to_string().contains("The `_geoRadius` filter expect three arguments: `_geoRadius(latitude, longitude, radius)`")); + + // georadius don't have any parameters + let result = FilterCondition::from_str(&rtxn, &index, "_geoRadius()"); + assert!(result.is_err()); + let error = result.unwrap_err(); + assert!(error.to_string().contains("The `_geoRadius` filter expect three arguments: `_geoRadius(latitude, longitude, radius)`")); + + // georadius don't have enough parameters + let result = FilterCondition::from_str(&rtxn, &index, "_geoRadius(1, 2)"); + assert!(result.is_err()); + let error = result.unwrap_err(); + assert!(error.to_string().contains("The `_geoRadius` filter expect three arguments: `_geoRadius(latitude, longitude, radius)`")); + + // georadius have too many parameters + let result = + FilterCondition::from_str(&rtxn, &index, "_geoRadius(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)"); + assert!(result.is_err()); + let error = result.unwrap_err(); + assert!(error.to_string().contains("The `_geoRadius` filter expect three arguments: `_geoRadius(latitude, longitude, radius)`")); + + // georadius have a bad latitude + let result = FilterCondition::from_str(&rtxn, &index, "_geoRadius(-200, 150, 10)"); + assert!(result.is_err()); + let error = result.unwrap_err(); + assert!(error + .to_string() + .contains("Latitude and longitude must be contained between -180 to 180 degrees.")); + + // georadius have a bad longitude + let result = FilterCondition::from_str(&rtxn, &index, "_geoRadius(-10, 181, 10)"); + assert!(result.is_err()); + let error = result.unwrap_err(); + assert!(error + .to_string() + .contains("Latitude and longitude must be contained between -180 to 180 degrees.")); + } + #[test] fn from_array() { let path = tempfile::tempdir().unwrap(); diff --git a/milli/src/search/facet/grammar.pest b/milli/src/search/facet/grammar.pest index 2096517d3..d07d5bca5 100644 --- a/milli/src/search/facet/grammar.pest +++ b/milli/src/search/facet/grammar.pest @@ -1,5 +1,5 @@ -key = _{quoted | word} -value = _{quoted | word} +key = _{reserved | quoted | word } +value = _{quoted | word } quoted = _{ (PUSH("'") | PUSH("\"")) ~ string ~ POP } string = {char*} word = ${(LETTER | NUMBER | "_" | "-" | ".")+} @@ -8,6 +8,9 @@ char = _{ !(PEEK | "\\") ~ ANY | "\\" ~ (PEEK | "\\" | "/" | "b" | "f" | "n" | "r" | "t") | "\\" ~ ("u" ~ ASCII_HEX_DIGIT{4})} +reserved = { "_geo" | "_geoDistance" | "_geoPoint" | ("_geoPoint" ~ parameters) } +// we deliberately choose to allow empty parameters to generate more specific error message later +parameters = {("(" ~ (value ~ ",")* ~ value? ~ ")") | ""} condition = _{between | eq | greater | less | geq | leq | neq} between = {key ~ value ~ "TO" ~ value} geq = {key ~ ">=" ~ value} @@ -16,10 +19,11 @@ neq = {key ~ "!=" ~ value} eq = {key ~ "=" ~ value} greater = {key ~ ">" ~ value} less = {key ~ "<" ~ value} +geo_radius = {"_geoRadius" ~ parameters } prgm = {SOI ~ expr ~ EOI} expr = _{ ( term ~ (operation ~ term)* ) } -term = { ("(" ~ expr ~ ")") | condition | not } +term = { ("(" ~ expr ~ ")") | condition | not | geo_radius } operation = _{ and | or } and = {"AND"} or = {"OR"} diff --git a/milli/src/search/mod.rs b/milli/src/search/mod.rs index 207f46f8a..f752f5822 100644 --- a/milli/src/search/mod.rs +++ b/milli/src/search/mod.rs @@ -148,13 +148,15 @@ impl<'a> Search<'a> { if let Some(sort_criteria) = &self.sort_criteria { let sortable_fields = self.index.sortable_fields(self.rtxn)?; for asc_desc in sort_criteria { - let field = asc_desc.field(); - if !sortable_fields.contains(field) { - return Err(UserError::InvalidSortableAttribute { - field: field.to_string(), - valid_fields: sortable_fields, + // we are not supposed to find any geoPoint in the criterion + if let Some(field) = asc_desc.field() { + if !sortable_fields.contains(field) { + return Err(UserError::InvalidSortableAttribute { + field: field.to_string(), + valid_fields: sortable_fields, + } + .into()); } - .into()); } } } diff --git a/milli/src/update/clear_documents.rs b/milli/src/update/clear_documents.rs index 789970a8e..e937cb65f 100644 --- a/milli/src/update/clear_documents.rs +++ b/milli/src/update/clear_documents.rs @@ -48,6 +48,8 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> { self.index.put_external_documents_ids(self.wtxn, &ExternalDocumentsIds::default())?; self.index.put_documents_ids(self.wtxn, &RoaringBitmap::default())?; self.index.put_field_distribution(self.wtxn, &FieldDistribution::default())?; + self.index.delete_geo_rtree(self.wtxn)?; + self.index.delete_geo_faceted_documents_ids(self.wtxn)?; // We clean all the faceted documents ids. let empty = RoaringBitmap::default(); @@ -93,7 +95,7 @@ mod tests { let content = &br#"[ { "id": 0, "name": "kevin", "age": 20 }, { "id": 1, "name": "kevina" }, - { "id": 2, "name": "benoit", "country": "France" } + { "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } } ]"#[..]; let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); builder.update_format(UpdateFormat::Json); @@ -107,13 +109,15 @@ mod tests { let rtxn = index.read_txn().unwrap(); - assert_eq!(index.fields_ids_map(&rtxn).unwrap().len(), 4); + assert_eq!(index.fields_ids_map(&rtxn).unwrap().len(), 5); assert!(index.words_fst(&rtxn).unwrap().is_empty()); assert!(index.words_prefixes_fst(&rtxn).unwrap().is_empty()); assert!(index.external_documents_ids(&rtxn).unwrap().is_empty()); assert!(index.documents_ids(&rtxn).unwrap().is_empty()); assert!(index.field_distribution(&rtxn).unwrap().is_empty()); + assert!(index.geo_rtree(&rtxn).unwrap().is_none()); + assert!(index.geo_faceted_documents_ids(&rtxn).unwrap().is_empty()); assert!(index.word_docids.is_empty(&rtxn).unwrap()); assert!(index.word_prefix_docids.is_empty(&rtxn).unwrap()); diff --git a/milli/src/update/delete_documents.rs b/milli/src/update/delete_documents.rs index 874eed6ee..b49cdc3cd 100644 --- a/milli/src/update/delete_documents.rs +++ b/milli/src/update/delete_documents.rs @@ -380,6 +380,24 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { drop(iter); + if let Some(mut rtree) = self.index.geo_rtree(self.wtxn)? { + let mut geo_faceted_doc_ids = self.index.geo_faceted_documents_ids(self.wtxn)?; + + let (points_to_remove, docids_to_remove): (Vec<_>, RoaringBitmap) = rtree + .iter() + .filter(|&point| self.documents_ids.contains(point.data)) + .cloned() + .map(|point| (point, point.data)) + .unzip(); + points_to_remove.iter().for_each(|point| { + rtree.remove(&point); + }); + geo_faceted_doc_ids -= docids_to_remove; + + self.index.put_geo_rtree(self.wtxn, &rtree)?; + self.index.put_geo_faceted_documents_ids(self.wtxn, &geo_faceted_doc_ids)?; + } + // We delete the documents ids that are under the facet field id values. remove_docids_from_facet_field_id_number_docids( self.wtxn, @@ -542,6 +560,8 @@ where #[cfg(test)] mod tests { + use std::collections::HashSet; + use big_s::S; use heed::EnvOpenOptions; use maplit::hashset; @@ -657,4 +677,95 @@ mod tests { wtxn.commit().unwrap(); } + + #[test] + fn delete_documents_with_geo_points() { + let path = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(10 * 1024 * 1024); // 10 MB + let index = Index::new(options, &path).unwrap(); + + let mut wtxn = index.write_txn().unwrap(); + let mut builder = Settings::new(&mut wtxn, &index, 0); + builder.set_primary_key(S("id")); + builder.set_filterable_fields(hashset!(S("_geo"))); + builder.set_sortable_fields(hashset!(S("_geo"))); + builder.execute(|_, _| ()).unwrap(); + + let content = &r#"[ + {"id":"1","city":"Lille", "_geo": { "lat": 50.629973371633746, "lng": 3.0569447399419570 } }, + {"id":"2","city":"Mons-en-Barœul", "_geo": { "lat": 50.641586120121050, "lng": 3.1106593480348670 } }, + {"id":"3","city":"Hellemmes", "_geo": { "lat": 50.631220965518080, "lng": 3.1106399673339933 } }, + {"id":"4","city":"Villeneuve-d'Ascq", "_geo": { "lat": 50.622468098014565, "lng": 3.1476425513437140 } }, + {"id":"5","city":"Hem", "_geo": { "lat": 50.655250871381355, "lng": 3.1897297266244130 } }, + {"id":"6","city":"Roubaix", "_geo": { "lat": 50.692473451896710, "lng": 3.1763326737747650 } }, + {"id":"7","city":"Tourcoing", "_geo": { "lat": 50.726397466736480, "lng": 3.1541653659578670 } }, + {"id":"8","city":"Mouscron", "_geo": { "lat": 50.745325554908610, "lng": 3.2206407854429853 } }, + {"id":"9","city":"Tournai", "_geo": { "lat": 50.605342528602630, "lng": 3.3758586941351414 } }, + {"id":"10","city":"Ghent", "_geo": { "lat": 51.053777403679035, "lng": 3.6957733119926930 } }, + {"id":"11","city":"Brussels", "_geo": { "lat": 50.846640974544690, "lng": 4.3370663564281840 } }, + {"id":"12","city":"Charleroi", "_geo": { "lat": 50.409570138889480, "lng": 4.4347354315085520 } }, + {"id":"13","city":"Mons", "_geo": { "lat": 50.450294178855420, "lng": 3.9623722870904690 } }, + {"id":"14","city":"Valenciennes", "_geo": { "lat": 50.351817774473545, "lng": 3.5326283646928800 } }, + {"id":"15","city":"Arras", "_geo": { "lat": 50.284487528579950, "lng": 2.7637515844478160 } }, + {"id":"16","city":"Cambrai", "_geo": { "lat": 50.179340577906700, "lng": 3.2189409952502930 } }, + {"id":"17","city":"Bapaume", "_geo": { "lat": 50.111276127236400, "lng": 2.8547894666083120 } }, + {"id":"18","city":"Amiens", "_geo": { "lat": 49.931472529669996, "lng": 2.2710499758317080 } }, + {"id":"19","city":"Compiègne", "_geo": { "lat": 49.444980887725656, "lng": 2.7913841281529015 } }, + {"id":"20","city":"Paris", "_geo": { "lat": 48.902100060895480, "lng": 2.3708400867406930 } } + ]"#[..]; + let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"]; + + let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + builder.update_format(UpdateFormat::Json); + builder.execute(content.as_bytes(), |_, _| ()).unwrap(); + + let external_document_ids = index.external_documents_ids(&wtxn).unwrap(); + let ids_to_delete: Vec = external_ids_to_delete + .iter() + .map(|id| external_document_ids.get(id.as_bytes()).unwrap()) + .collect(); + + // Delete some documents. + let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap(); + external_ids_to_delete.iter().for_each(|id| drop(builder.delete_external_id(id))); + builder.execute().unwrap(); + + wtxn.commit().unwrap(); + + let rtxn = index.read_txn().unwrap(); + let rtree = index.geo_rtree(&rtxn).unwrap().unwrap(); + let geo_faceted_doc_ids = index.geo_faceted_documents_ids(&rtxn).unwrap(); + + let all_geo_ids = rtree.iter().map(|point| point.data).collect::>(); + let all_geo_documents = index + .documents(&rtxn, all_geo_ids.iter().copied()) + .unwrap() + .iter() + .map(|(id, _)| *id) + .collect::>(); + + let all_geo_faceted_ids = geo_faceted_doc_ids.iter().collect::>(); + let all_geo_faceted_documents = index + .documents(&rtxn, all_geo_faceted_ids.iter().copied()) + .unwrap() + .iter() + .map(|(id, _)| *id) + .collect::>(); + + assert_eq!( + all_geo_documents, all_geo_faceted_documents, + "There is an inconsistency between the geo_faceted database and the rtree" + ); + + for id in all_geo_documents.iter() { + assert!(!ids_to_delete.contains(&id), "The document {} was supposed to be deleted", id); + } + + assert_eq!( + all_geo_ids.len(), + all_geo_documents.len(), + "We deleted documents that were not supposed to be deleted" + ); + } } diff --git a/milli/src/update/index_documents/extract/extract_geo_points.rs b/milli/src/update/index_documents/extract/extract_geo_points.rs new file mode 100644 index 000000000..a36b608ee --- /dev/null +++ b/milli/src/update/index_documents/extract/extract_geo_points.rs @@ -0,0 +1,44 @@ +use std::fs::File; +use std::io; + +use concat_arrays::concat_arrays; +use serde_json::Value; + +use super::helpers::{create_writer, writer_into_reader, GrenadParameters}; +use crate::{FieldId, InternalError, Result, UserError}; + +/// Extracts the geographical coordinates contained in each document under the `_geo` field. +/// +/// Returns the generated grenad reader containing the docid as key associated to the (latitude, longitude) +pub fn extract_geo_points( + mut obkv_documents: grenad::Reader, + indexer: GrenadParameters, + primary_key_id: FieldId, + geo_field_id: FieldId, +) -> Result> { + let mut writer = tempfile::tempfile().and_then(|file| { + create_writer(indexer.chunk_compression_type, indexer.chunk_compression_level, file) + })?; + + while let Some((docid_bytes, value)) = obkv_documents.next()? { + let obkv = obkv::KvReader::new(value); + let point: Value = match obkv.get(geo_field_id) { + Some(point) => serde_json::from_slice(point).map_err(InternalError::SerdeJson)?, + None => continue, + }; + + if let Some((lat, lng)) = point["lat"].as_f64().zip(point["lng"].as_f64()) { + // this will create an array of 16 bytes (two 8 bytes floats) + let bytes: [u8; 16] = concat_arrays![lat.to_ne_bytes(), lng.to_ne_bytes()]; + writer.insert(docid_bytes, bytes)?; + } else { + // All document must have a primary key so we can unwrap safely here + let primary_key = obkv.get(primary_key_id).unwrap(); + let primary_key = + serde_json::from_slice(primary_key).map_err(InternalError::SerdeJson)?; + Err(UserError::InvalidGeoField { document_id: primary_key, object: point })? + } + } + + Ok(writer_into_reader(writer)?) +} diff --git a/milli/src/update/index_documents/extract/mod.rs b/milli/src/update/index_documents/extract/mod.rs index bb49e3e51..47a62be67 100644 --- a/milli/src/update/index_documents/extract/mod.rs +++ b/milli/src/update/index_documents/extract/mod.rs @@ -3,6 +3,7 @@ mod extract_facet_number_docids; mod extract_facet_string_docids; mod extract_fid_docid_facet_values; mod extract_fid_word_count_docids; +mod extract_geo_points; mod extract_word_docids; mod extract_word_level_position_docids; mod extract_word_pair_proximity_docids; @@ -19,6 +20,7 @@ use self::extract_facet_number_docids::extract_facet_number_docids; use self::extract_facet_string_docids::extract_facet_string_docids; use self::extract_fid_docid_facet_values::extract_fid_docid_facet_values; use self::extract_fid_word_count_docids::extract_fid_word_count_docids; +use self::extract_geo_points::extract_geo_points; use self::extract_word_docids::extract_word_docids; use self::extract_word_level_position_docids::extract_word_level_position_docids; use self::extract_word_pair_proximity_docids::extract_word_pair_proximity_docids; @@ -37,6 +39,8 @@ pub(crate) fn data_from_obkv_documents( lmdb_writer_sx: Sender>, searchable_fields: Option>, faceted_fields: HashSet, + primary_key_id: FieldId, + geo_field_id: Option, stop_words: Option>, ) -> Result<()> { let result: Result<(Vec<_>, (Vec<_>, Vec<_>))> = obkv_chunks @@ -48,6 +52,8 @@ pub(crate) fn data_from_obkv_documents( lmdb_writer_sx.clone(), &searchable_fields, &faceted_fields, + primary_key_id, + geo_field_id, &stop_words, ) }) @@ -168,6 +174,8 @@ fn extract_documents_data( lmdb_writer_sx: Sender>, searchable_fields: &Option>, faceted_fields: &HashSet, + primary_key_id: FieldId, + geo_field_id: Option, stop_words: &Option>, ) -> Result<( grenad::Reader, @@ -177,6 +185,19 @@ fn extract_documents_data( let _ = lmdb_writer_sx.send(Ok(TypedChunk::Documents(documents_chunk.clone()))); + if let Some(geo_field_id) = geo_field_id { + let documents_chunk_cloned = documents_chunk.clone(); + let lmdb_writer_sx_cloned = lmdb_writer_sx.clone(); + rayon::spawn(move || { + let result = + extract_geo_points(documents_chunk_cloned, indexer, primary_key_id, geo_field_id); + let _ = match result { + Ok(geo_points) => lmdb_writer_sx_cloned.send(Ok(TypedChunk::GeoPoints(geo_points))), + Err(error) => lmdb_writer_sx_cloned.send(Err(error)), + }; + }); + } + let (docid_word_positions_chunk, docid_fid_facet_values_chunks): (Result<_>, Result<_>) = rayon::join( || { diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 861a5489e..bdd00dc56 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -228,11 +228,27 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { Receiver>, ) = crossbeam_channel::unbounded(); + // get the primary key field id + let primary_key_id = fields_ids_map.id(&primary_key).unwrap(); + // get searchable fields for word databases let searchable_fields = self.index.searchable_fields_ids(self.wtxn)?.map(HashSet::from_iter); // get filterable fields for facet databases let faceted_fields = self.index.faceted_fields_ids(self.wtxn)?; + // get the fid of the `_geo` field. + let geo_field_id = match self.index.fields_ids_map(self.wtxn)?.id("_geo") { + Some(gfid) => { + let is_sortable = self.index.sortable_fields_ids(self.wtxn)?.contains(&gfid); + let is_filterable = self.index.filterable_fields_ids(self.wtxn)?.contains(&gfid); + if is_sortable || is_filterable { + Some(gfid) + } else { + None + } + } + None => None, + }; let stop_words = self.index.stop_words(self.wtxn)?; // let stop_words = stop_words.as_ref(); @@ -261,6 +277,8 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { lmdb_writer_sx.clone(), searchable_fields, faceted_fields, + primary_key_id, + geo_field_id, stop_words, ) }); @@ -876,12 +894,12 @@ mod tests { // First we send 3 documents with an id for only one of them. let mut wtxn = index.write_txn().unwrap(); let documents = &r#"[ - { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance", "price": 3.5 }, + { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance", "price": 3.5, "_geo": { "lat": 12, "lng": 42 } }, { "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" , "price": 10.0 }, { "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy", "price": 25.99 }, { "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" }, { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, - { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams" } + { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "_geo": { "lat": 35, "lng": 23 } } ]"#[..]; let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); builder.update_format(UpdateFormat::Json); @@ -917,7 +935,7 @@ mod tests { { "objectId": 123, "title": "Pride and Prejudice", "comment": "A great book" }, { "objectId": 456, "title": "Le Petit Prince", "comment": "A french book" }, { "objectId": 1, "title": "Alice In Wonderland", "comment": "A weird book" }, - { "objectId": 30, "title": "Hamlet" } + { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } ]"#[..]; let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); builder.update_format(UpdateFormat::Json); @@ -934,7 +952,7 @@ mod tests { assert!(external_documents_ids.get("30").is_none()); let content = &br#"[ - { "objectId": 30, "title": "Hamlet" } + { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } ]"#[..]; let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); builder.update_format(UpdateFormat::Json); @@ -944,7 +962,7 @@ mod tests { assert!(external_documents_ids.get("30").is_some()); let content = &br#"[ - { "objectId": 30, "title": "Hamlet" } + { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } ]"#[..]; let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); builder.update_format(UpdateFormat::Json); diff --git a/milli/src/update/index_documents/typed_chunk.rs b/milli/src/update/index_documents/typed_chunk.rs index 5f28034fe..b17f28b66 100644 --- a/milli/src/update/index_documents/typed_chunk.rs +++ b/milli/src/update/index_documents/typed_chunk.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::convert::TryInto; use std::fs::File; use heed::types::ByteSlice; @@ -6,11 +7,12 @@ use heed::{BytesDecode, RwTxn}; use roaring::RoaringBitmap; use super::helpers::{ - roaring_bitmap_from_u32s_array, serialize_roaring_bitmap, valid_lmdb_key, CursorClonableMmap, + self, roaring_bitmap_from_u32s_array, serialize_roaring_bitmap, valid_lmdb_key, + CursorClonableMmap, }; use crate::heed_codec::facet::{decode_prefix_string, encode_prefix_string}; use crate::update::index_documents::helpers::into_clonable_grenad; -use crate::{BoRoaringBitmapCodec, CboRoaringBitmapCodec, Index, Result}; +use crate::{BoRoaringBitmapCodec, CboRoaringBitmapCodec, DocumentId, GeoPoint, Index, Result}; pub(crate) enum TypedChunk { DocidWordPositions(grenad::Reader), @@ -24,6 +26,7 @@ pub(crate) enum TypedChunk { WordPairProximityDocids(grenad::Reader), FieldIdFacetStringDocids(grenad::Reader), FieldIdFacetNumberDocids(grenad::Reader), + GeoPoints(grenad::Reader), } /// Write typed chunk in the corresponding LMDB database of the provided index. @@ -177,6 +180,24 @@ pub(crate) fn write_typed_chunk_into_index( )?; is_merged_database = true; } + TypedChunk::GeoPoints(mut geo_points) => { + let mut rtree = index.geo_rtree(wtxn)?.unwrap_or_default(); + let mut geo_faceted_docids = index.geo_faceted_documents_ids(wtxn)?; + + while let Some((key, value)) = geo_points.next()? { + // convert the key back to a u32 (4 bytes) + let docid = key.try_into().map(DocumentId::from_be_bytes).unwrap(); + + // convert the latitude and longitude back to a f64 (8 bytes) + let (lat, tail) = helpers::try_split_array_at::(value).unwrap(); + let (lng, _) = helpers::try_split_array_at::(tail).unwrap(); + let point = [f64::from_ne_bytes(lat), f64::from_ne_bytes(lng)]; + rtree.insert(GeoPoint::new(point, docid)); + geo_faceted_docids.insert(docid); + } + index.put_geo_rtree(wtxn, &rtree)?; + index.put_geo_faceted_documents_ids(wtxn, &geo_faceted_docids)?; + } } Ok((RoaringBitmap::new(), is_merged_database)) diff --git a/milli/tests/assets/test_set.ndjson b/milli/tests/assets/test_set.ndjson index 89d9f1109..9a0fe5b0a 100644 --- a/milli/tests/assets/test_set.ndjson +++ b/milli/tests/assets/test_set.ndjson @@ -1,17 +1,17 @@ -{"id":"A","word_rank":0,"typo_rank":1,"proximity_rank":15,"attribute_rank":505,"exact_rank":5,"asc_desc_rank":0,"sort_by_rank":0,"title":"hell o","description":"hell o is the fourteenth episode of the american television series glee performing songs with this word","tag":"blue","":""} -{"id":"B","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":1,"sort_by_rank":2,"title":"hello","description":"hello is a song recorded by english singer songwriter adele","tag":"red","":""} -{"id":"C","word_rank":0,"typo_rank":1,"proximity_rank":8,"attribute_rank":336,"exact_rank":4,"asc_desc_rank":2,"sort_by_rank":0,"title":"hell on earth","description":"hell on earth is the third studio album by american hip hop duo mobb deep","tag":"blue","":""} -{"id":"D","word_rank":0,"typo_rank":1,"proximity_rank":10,"attribute_rank":757,"exact_rank":4,"asc_desc_rank":3,"sort_by_rank":2,"title":"hell on wheels tv series","description":"the construction of the first transcontinental railroad across the united states in the world","tag":"red","":""} -{"id":"E","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":4,"sort_by_rank":1,"title":"hello kitty","description":"also known by her full name kitty white is a fictional character produced by the japanese company sanrio","tag":"green","":""} -{"id":"F","word_rank":2,"typo_rank":1,"proximity_rank":0,"attribute_rank":1017,"exact_rank":5,"asc_desc_rank":5,"sort_by_rank":0,"title":"laptop orchestra","description":"a laptop orchestra lork or lo is a chamber music ensemble consisting primarily of laptops like helo huddersfield experimental laptop orchestra","tag":"blue","":""} -{"id":"G","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":5,"sort_by_rank":2,"title":"hello world film","description":"hello world is a 2019 japanese animated sci fi romantic drama film directed by tomohiko ito and produced by graphinica","tag":"red","":""} -{"id":"H","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":4,"sort_by_rank":1,"title":"world hello day","description":"holiday observed on november 21 to express that conflicts should be resolved through communication rather than the use of force","tag":"green","":""} -{"id":"I","word_rank":0,"typo_rank":0,"proximity_rank":8,"attribute_rank":338,"exact_rank":3,"asc_desc_rank":3,"sort_by_rank":0,"title":"hello world song","description":"hello world is a song written by tom douglas tony lane and david lee and recorded by american country music group lady antebellum","tag":"blue","":""} -{"id":"J","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":1,"exact_rank":3,"asc_desc_rank":2,"sort_by_rank":1,"title":"hello cruel world","description":"hello cruel world is an album by new zealand band tall dwarfs","tag":"green","":""} -{"id":"K","word_rank":0,"typo_rank":2,"proximity_rank":9,"attribute_rank":670,"exact_rank":5,"asc_desc_rank":1,"sort_by_rank":2,"title":"ello creation system","description":"in few word ello was a construction toy created by the american company mattel to engage girls in construction play","tag":"red","":""} -{"id":"L","word_rank":0,"typo_rank":0,"proximity_rank":2,"attribute_rank":250,"exact_rank":4,"asc_desc_rank":0,"sort_by_rank":0,"title":"good morning world","description":"good morning world is an american sitcom broadcast on cbs tv during the 1967 1968 season","tag":"blue","":""} -{"id":"M","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":0,"asc_desc_rank":0,"sort_by_rank":2,"title":"hello world america","description":"a perfect match for a perfect engine using the query hello world america","tag":"red","":""} -{"id":"N","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":1,"asc_desc_rank":4,"sort_by_rank":1,"title":"hello world america unleashed","description":"a very good match for a very good engine using the query hello world america","tag":"green","":""} -{"id":"O","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":10,"exact_rank":0,"asc_desc_rank":6,"sort_by_rank":0,"title":"a perfect match for a perfect engine using the query hello world america","description":"hello world america","tag":"blue","":""} -{"id":"P","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":12,"exact_rank":1,"asc_desc_rank":3,"sort_by_rank":2,"title":"a very good match for a very good engine using the query hello world america","description":"hello world america unleashed","tag":"red","":""} -{"id":"Q","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":2,"sort_by_rank":1,"title":"hello world","description":"a hello world program generally is a computer program that outputs or displays the message hello world","tag":"green","":""} +{"id":"A","word_rank":0,"typo_rank":1,"proximity_rank":15,"attribute_rank":505,"exact_rank":5,"asc_desc_rank":0,"sort_by_rank":0,"geo_rank":43,"title":"hell o","description":"hell o is the fourteenth episode of the american television series glee performing songs with this word","tag":"blue","_geo": { "lat": 50.62984446145472, "lng": 3.085712705162039 },"":""} +{"id":"B","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":1,"sort_by_rank":2,"geo_rank":191,"title":"hello","description":"hello is a song recorded by english singer songwriter adele","tag":"red","_geo": { "lat": 50.63047567664291, "lng": 3.088852230809636 },"":""} +{"id":"C","word_rank":0,"typo_rank":1,"proximity_rank":8,"attribute_rank":336,"exact_rank":4,"asc_desc_rank":2,"sort_by_rank":0,"geo_rank":283,"title":"hell on earth","description":"hell on earth is the third studio album by american hip hop duo mobb deep","tag":"blue","_geo": { "lat": 50.6321800003937, "lng": 3.088331882262139 },"":""} +{"id":"D","word_rank":0,"typo_rank":1,"proximity_rank":10,"attribute_rank":757,"exact_rank":4,"asc_desc_rank":3,"sort_by_rank":2,"geo_rank":1381,"title":"hell on wheels tv series","description":"the construction of the first transcontinental railroad across the united states in the world","tag":"red","_geo": { "lat": 50.63728851135729, "lng": 3.0703951595971626 },"":""} +{"id":"E","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":1979,"title":"hello kitty","description":"also known by her full name kitty white is a fictional character produced by the japanese company sanrio","tag":"green","_geo": { "lat": 50.64264610511925, "lng": 3.0665099941857634 },"":""} +{"id":"F","word_rank":2,"typo_rank":1,"proximity_rank":0,"attribute_rank":1017,"exact_rank":5,"asc_desc_rank":5,"sort_by_rank":0,"geo_rank":65022,"title":"laptop orchestra","description":"a laptop orchestra lork or lo is a chamber music ensemble consisting primarily of laptops like helo huddersfield experimental laptop orchestra","tag":"blue","_geo": { "lat": 51.05028653642387, "lng": 3.7301072771642096 },"":""} +{"id":"G","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":5,"sort_by_rank":2,"geo_rank":34692,"title":"hello world film","description":"hello world is a 2019 japanese animated sci fi romantic drama film directed by tomohiko ito and produced by graphinica","tag":"red","_geo": { "lat": 50.78776041427129, "lng": 2.661201766290338 },"":""} +{"id":"H","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":202182,"title":"world hello day","description":"holiday observed on november 21 to express that conflicts should be resolved through communication rather than the use of force","tag":"green","_geo": { "lat": 48.875617484531965, "lng": 2.346747821504194 },"":""} +{"id":"I","word_rank":0,"typo_rank":0,"proximity_rank":8,"attribute_rank":338,"exact_rank":3,"asc_desc_rank":3,"sort_by_rank":0,"geo_rank":740667,"title":"hello world song","description":"hello world is a song written by tom douglas tony lane and david lee and recorded by american country music group lady antebellum","tag":"blue","_geo": { "lat": 43.973998070351065, "lng": 3.4661837318345032 },"":""} +{"id":"J","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":1,"exact_rank":3,"asc_desc_rank":2,"sort_by_rank":1,"geo_rank":739020,"title":"hello cruel world","description":"hello cruel world is an album by new zealand band tall dwarfs","tag":"green","_geo": { "lat": 43.98920130353838, "lng": 3.480519311627928 },"":""} +{"id":"K","word_rank":0,"typo_rank":2,"proximity_rank":9,"attribute_rank":670,"exact_rank":5,"asc_desc_rank":1,"sort_by_rank":2,"geo_rank":738830,"title":"ello creation system","description":"in few word ello was a construction toy created by the american company mattel to engage girls in construction play","tag":"red","_geo": { "lat": 43.99155030238669, "lng": 3.503453528249425 },"":""} +{"id":"L","word_rank":0,"typo_rank":0,"proximity_rank":2,"attribute_rank":250,"exact_rank":4,"asc_desc_rank":0,"sort_by_rank":0,"geo_rank":737861,"title":"good morning world","description":"good morning world is an american sitcom broadcast on cbs tv during the 1967 1968 season","tag":"blue","_geo": { "lat": 44.000507750283695, "lng": 3.5116812040621572 },"":""} +{"id":"M","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":0,"asc_desc_rank":0,"sort_by_rank":2,"geo_rank":739203,"title":"hello world america","description":"a perfect match for a perfect engine using the query hello world america","tag":"red","_geo": { "lat": 43.99150729038736, "lng": 3.606143957295055 },"":""} +{"id":"N","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":1,"asc_desc_rank":4,"sort_by_rank":1,"geo_rank":9499586,"title":"hello world america unleashed","description":"a very good match for a very good engine using the query hello world america","tag":"green","_geo": { "lat": 35.511540843367115, "lng": 138.764368875787 },"":""} +{"id":"O","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":10,"exact_rank":0,"asc_desc_rank":6,"sort_by_rank":0,"geo_rank":9425163,"title":"a perfect match for a perfect engine using the query hello world america","description":"hello world america","tag":"blue","_geo": { "lat": 35.00536702277189, "lng": 135.76118763940391 },"":""} +{"id":"P","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":12,"exact_rank":1,"asc_desc_rank":3,"sort_by_rank":2,"geo_rank":9422437,"title":"a very good match for a very good engine using the query hello world america","description":"hello world america unleashed","tag":"red","_geo": { "lat": 35.06462306367058, "lng": 135.8338440354251 },"":""} +{"id":"Q","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":2,"sort_by_rank":1,"geo_rank":9339230,"title":"hello world","description":"a hello world program generally is a computer program that outputs or displays the message hello world","tag":"green","_geo": { "lat": 34.39548365683149, "lng": 132.4535960928883 },"":""} diff --git a/milli/tests/search/filters.rs b/milli/tests/search/filters.rs index c810b47af..d992a8e95 100644 --- a/milli/tests/search/filters.rs +++ b/milli/tests/search/filters.rs @@ -47,6 +47,11 @@ test_filter!(eq_mix_and_filter, vec![Right("tag=red"), Right("asc_desc_rank=1")] test_filter!(eq_string_or_filter, vec![Left(vec!["tag=red", "tag=green"])]); test_filter!(eq_mix_or_filter, vec![Left(vec!["tag=red", "asc_desc_rank=1"])]); test_filter!(eq_number_or_filter, vec![Left(vec!["asc_desc_rank=3", "asc_desc_rank=1"])]); +test_filter!(geo_radius, vec![Right("_geoRadius(50.630010347667806, 3.086251829166809, 100000)")]); +test_filter!( + not_geo_radius, + vec![Right("NOT _geoRadius(50.630010347667806, 3.086251829166809, 1000000)")] +); test_filter!(eq_complex_filter, vec![Left(vec!["tag=red", "tag=green"]), Right("asc_desc_rank=3")]); test_filter!( eq_complex_filter_2, diff --git a/milli/tests/search/mod.rs b/milli/tests/search/mod.rs index c34434c4e..e3f6c5b09 100644 --- a/milli/tests/search/mod.rs +++ b/milli/tests/search/mod.rs @@ -6,7 +6,7 @@ use either::{Either, Left, Right}; use heed::EnvOpenOptions; use maplit::{hashmap, hashset}; use milli::update::{Settings, UpdateBuilder, UpdateFormat}; -use milli::{AscDesc, Criterion, DocumentId, Index}; +use milli::{AscDesc, Criterion, DocumentId, Index, Member}; use serde::Deserialize; use slice_group_by::GroupBy; @@ -37,6 +37,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { builder.set_filterable_fields(hashset! { S("tag"), S("asc_desc_rank"), + S("_geo"), }); builder.set_sortable_fields(hashset! { S("tag"), @@ -99,11 +100,11 @@ pub fn expected_order( new_groups .extend(group.linear_group_by_key(|d| d.proximity_rank).map(Vec::from)); } - Criterion::Sort if sort_by == [AscDesc::Asc(S("tag"))] => { + Criterion::Sort if sort_by == [AscDesc::Asc(Member::Field(S("tag")))] => { group.sort_by_key(|d| d.sort_by_rank); new_groups.extend(group.linear_group_by_key(|d| d.sort_by_rank).map(Vec::from)); } - Criterion::Sort if sort_by == [AscDesc::Desc(S("tag"))] => { + Criterion::Sort if sort_by == [AscDesc::Desc(Member::Field(S("tag")))] => { group.sort_by_key(|d| Reverse(d.sort_by_rank)); new_groups.extend(group.linear_group_by_key(|d| d.sort_by_rank).map(Vec::from)); } @@ -162,6 +163,10 @@ fn execute_filter(filter: &str, document: &TestDocument) -> Option { if document.asc_desc_rank > filter.parse().unwrap() { id = Some(document.id.clone()) } + } else if filter.starts_with("_geoRadius") { + id = (document.geo_rank < 100000).then(|| document.id.clone()); + } else if filter.starts_with("NOT _geoRadius") { + id = (document.geo_rank > 1000000).then(|| document.id.clone()); } id } @@ -205,6 +210,7 @@ pub struct TestDocument { pub exact_rank: u32, pub asc_desc_rank: u32, pub sort_by_rank: u32, + pub geo_rank: u32, pub title: String, pub description: String, pub tag: String, diff --git a/milli/tests/search/query_criteria.rs b/milli/tests/search/query_criteria.rs index c9720d652..f6a937f67 100644 --- a/milli/tests/search/query_criteria.rs +++ b/milli/tests/search/query_criteria.rs @@ -5,7 +5,7 @@ use heed::EnvOpenOptions; use itertools::Itertools; use maplit::hashset; use milli::update::{Settings, UpdateBuilder, UpdateFormat}; -use milli::{AscDesc, Criterion, Index, Search, SearchResult}; +use milli::{AscDesc, Criterion, Index, Member, Search, SearchResult}; use rand::Rng; use Criterion::*; @@ -163,28 +163,28 @@ test_criterion!( DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, vec![Sort], - vec![AscDesc::Asc(S("tag"))] + vec![AscDesc::Asc(Member::Field(S("tag")))] ); test_criterion!( sort_by_asc_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, vec![Sort], - vec![AscDesc::Asc(S("tag"))] + vec![AscDesc::Asc(Member::Field(S("tag")))] ); test_criterion!( sort_by_desc_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, vec![Sort], - vec![AscDesc::Desc(S("tag"))] + vec![AscDesc::Desc(Member::Field(S("tag")))] ); test_criterion!( sort_by_desc_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, vec![Sort], - vec![AscDesc::Desc(S("tag"))] + vec![AscDesc::Desc(Member::Field(S("tag")))] ); test_criterion!( default_criteria_order, diff --git a/milli/tests/search/sort.rs b/milli/tests/search/sort.rs index fe87f0623..86404bb99 100644 --- a/milli/tests/search/sort.rs +++ b/milli/tests/search/sort.rs @@ -1,6 +1,6 @@ use big_s::S; use milli::Criterion::{Attribute, Exactness, Proximity, Typo, Words}; -use milli::{AscDesc, Error, Search, UserError}; +use milli::{AscDesc, Error, Member, Search, UserError}; use crate::search::{self, EXTERNAL_DOCUMENTS_IDS}; @@ -16,7 +16,7 @@ fn sort_ranking_rule_missing() { search.limit(EXTERNAL_DOCUMENTS_IDS.len()); search.authorize_typos(true); search.optional_words(true); - search.sort_criteria(vec![AscDesc::Asc(S("tag"))]); + search.sort_criteria(vec![AscDesc::Asc(Member::Field(S("tag")))]); let result = search.execute(); assert!(matches!(result, Err(Error::UserError(UserError::SortRankingRuleMissing))));