Merge #197

197: Update milli (v0.3.1) with filterable attributes r=MarinPostma a=curquiza

Fixes #187 and #70
Also fixes #195 

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

Cargo.lock

@@ -1660,14 +1660,13 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "0.2.1"
+version = "0.3.1"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.2.1#25f75d4d03732131e6edcf20f4d126210b159d43"
+source = "git+https://github.com/meilisearch/milli.git?tag=v0.3.1#bc020317935da4ea08061b3d4518cbbd40184856"
 dependencies = [
  "anyhow",
  "bstr",
  "byteorder",
  "chrono",
- "crossbeam-channel",
  "csv",
  "either",
  "flate2",

meilisearch-http/Cargo.toml

@@ -51,7 +51,7 @@ main_error = "0.1.0"
 meilisearch-error = { path = "../meilisearch-error" }
 meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" }
 memmap = "0.7.0"
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.2.1" }
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.3.1" }
 mime = "0.3.16"
 once_cell = "1.5.2"
 oxidized-json-checker = "0.3.2"

meilisearch-http/src/index/mod.rs

@@ -67,7 +67,6 @@ impl Index {
         let faceted_attributes = self
             .faceted_fields(&txn)?
             .into_iter()
-            .map(|(k, v)| (k, v.to_string()))
             .collect();
 
         let criteria = self
@@ -83,15 +82,15 @@ impl Index {
             })
             .transpose()?
             .unwrap_or_else(BTreeSet::new);
-        let distinct_attribute = self.distinct_attribute(&txn)?.map(String::from);
+        let distinct_field = self.distinct_field(&txn)?.map(String::from);
 
         Ok(Settings {
             displayed_attributes: Some(displayed_attributes),
             searchable_attributes: Some(searchable_attributes),
-            attributes_for_faceting: Some(Some(faceted_attributes)),
+            filterable_attributes: Some(Some(faceted_attributes)),
             ranking_rules: Some(Some(criteria)),
             stop_words: Some(Some(stop_words)),
-            distinct_attribute: Some(distinct_attribute),
+            distinct_attribute: Some(distinct_field),
             _kind: PhantomData,
         })
     }

meilisearch-http/src/index/search.rs

@@ -8,7 +8,7 @@ use heed::RoTxn;
 use indexmap::IndexMap;
 use itertools::Itertools;
 use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
-use milli::{facet::FacetValue, FacetCondition, FieldId, FieldsIdsMap, MatchingWords};
+use milli::{FilterCondition, FieldId, FieldsIdsMap, MatchingWords};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
@@ -57,7 +57,7 @@ pub struct SearchResult {
     pub offset: usize,
     pub processing_time_ms: u128,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub facet_distributions: Option<BTreeMap<String, BTreeMap<FacetValue, u64>>>,
+    pub facet_distributions: Option<BTreeMap<String, BTreeMap<String, u64>>>,
 }
 
 impl Index {
@@ -76,7 +76,7 @@ impl Index {
 
         if let Some(ref filter) = query.filter {
             if let Some(facets) = parse_facets(filter, self, &rtxn)? {
-                search.facet_condition(facets);
+                search.filter(facets);
             }
         }
 
@@ -272,7 +272,7 @@ impl Matcher for HashSet<String> {
 
 impl Matcher for MatchingWords {
     fn matches(&self, w: &str) -> bool {
-        self.matches(w)
+        self.matching_bytes(w).is_some()
     }
 }
 
@@ -335,9 +335,9 @@ fn parse_facets(
     facets: &Value,
     index: &Index,
     txn: &RoTxn,
-) -> anyhow::Result<Option<FacetCondition>> {
+) -> anyhow::Result<Option<FilterCondition>> {
     match facets {
-        Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)),
+        Value::String(expr) => Ok(Some(FilterCondition::from_str(txn, index, expr)?)),
         Value::Array(arr) => parse_facets_array(txn, index, arr),
         v => bail!("Invalid facet expression, expected Array, found: {:?}", v),
     }
@@ -347,7 +347,7 @@ fn parse_facets_array(
     txn: &RoTxn,
     index: &Index,
     arr: &[Value],
-) -> anyhow::Result<Option<FacetCondition>> {
+) -> anyhow::Result<Option<FilterCondition>> {
     let mut ands = Vec::new();
     for value in arr {
         match value {
@@ -369,7 +369,7 @@ fn parse_facets_array(
         }
     }
 
-    FacetCondition::from_array(txn, &index.0, ands)
+    FilterCondition::from_array(txn, &index.0, ands)
 }
 
 #[cfg(test)]

meilisearch-http/src/index/updates.rs

@@ -1,4 +1,4 @@
-use std::collections::{BTreeSet, HashMap};
+use std::collections::{BTreeSet, HashSet};
 use std::io;
 use std::marker::PhantomData;
 use std::num::NonZeroUsize;
@@ -51,7 +51,7 @@ pub struct Settings<T> {
         deserialize_with = "deserialize_some",
         skip_serializing_if = "Option::is_none"
     )]
-    pub attributes_for_faceting: Option<Option<HashMap<String, String>>>,
+    pub filterable_attributes: Option<Option<HashSet<String>>>,
 
     #[serde(
         default,
@@ -81,7 +81,7 @@ impl Settings<Checked> {
         Settings {
             displayed_attributes: Some(None),
             searchable_attributes: Some(None),
-            attributes_for_faceting: Some(None),
+            filterable_attributes: Some(None),
             ranking_rules: Some(None),
             stop_words: Some(None),
             distinct_attribute: Some(None),
@@ -93,7 +93,7 @@ impl Settings<Checked> {
         let Self {
             displayed_attributes,
             searchable_attributes,
-            attributes_for_faceting,
+            filterable_attributes,
             ranking_rules,
             stop_words,
             distinct_attribute,
@@ -103,7 +103,7 @@ impl Settings<Checked> {
         Settings {
             displayed_attributes,
             searchable_attributes,
-            attributes_for_faceting,
+            filterable_attributes,
             ranking_rules,
             stop_words,
             distinct_attribute,
@@ -139,7 +139,7 @@ impl Settings<Unchecked> {
         Settings {
             displayed_attributes,
             searchable_attributes,
-            attributes_for_faceting: self.attributes_for_faceting,
+            filterable_attributes: self.filterable_attributes,
             ranking_rules: self.ranking_rules,
             stop_words: self.stop_words,
             distinct_attribute: self.distinct_attribute,
@@ -252,9 +252,9 @@ impl Index {
             }
         }
 
-        if let Some(ref facet_types) = settings.attributes_for_faceting {
+        if let Some(ref facet_types) = settings.filterable_attributes {
-            let facet_types = facet_types.clone().unwrap_or_else(HashMap::new);
+            let facet_types = facet_types.clone().unwrap_or_else(HashSet::new);
-            builder.set_faceted_fields(facet_types);
+            builder.set_filterable_fields(facet_types);
         }
 
         if let Some(ref criteria) = settings.ranking_rules {
@@ -273,8 +273,8 @@ impl Index {
 
         if let Some(ref distinct_attribute) = settings.distinct_attribute {
             match distinct_attribute {
-                Some(attr) => builder.set_distinct_attribute(attr.clone()),
+                Some(attr) => builder.set_distinct_field(attr.clone()),
-                None => builder.reset_distinct_attribute(),
+                None => builder.reset_distinct_field(),
             }
         }
 
@@ -329,7 +329,7 @@ mod test {
         let settings = Settings {
             displayed_attributes: Some(Some(vec![String::from("hello")])),
             searchable_attributes: Some(Some(vec![String::from("hello")])),
-            attributes_for_faceting: None,
+            filterable_attributes: None,
             ranking_rules: None,
             stop_words: None,
             distinct_attribute: None,
@@ -348,7 +348,7 @@ mod test {
         let settings = Settings {
             displayed_attributes: Some(Some(vec![String::from("*")])),
             searchable_attributes: Some(Some(vec![String::from("hello"), String::from("*")])),
-            attributes_for_faceting: None,
+            filterable_attributes: None,
             ranking_rules: None,
             stop_words: None,
             distinct_attribute: None,

meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs

@@ -73,7 +73,7 @@ struct Settings {
     #[serde(default, deserialize_with = "deserialize_some")]
     pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
     #[serde(default, deserialize_with = "deserialize_some")]
-    pub attributes_for_faceting: Option<Option<Vec<String>>>,
+    pub filterable_attributes: Option<Option<Vec<String>>>,
 }
 
 fn load_index(
@@ -145,7 +145,7 @@ impl From<Settings> for index_controller::Settings<Unchecked> {
             // representing the name of the faceted field + the type of the field. Since the type
             // was not known in the V1 of the dump we are just going to assume everything is a
             // String
-            attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().map(|key| (key, String::from("string"))).collect())),
+            filterable_attributes: settings.filterable_attributes.map(|o| o.map(|vec| vec.into_iter().collect())),
             // we need to convert the old `Vec<String>` into a `BTreeSet<String>`
             ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| {
                 match criterion.as_str() {

meilisearch-http/src/routes/settings/mod.rs

@@ -74,9 +74,9 @@ macro_rules! make_setting_route {
 }
 
 make_setting_route!(
-    "/indexes/{index_uid}/settings/attributes-for-faceting",
+    "/indexes/{index_uid}/settings/filterable-attributes",
-    std::collections::HashMap<String, String>,
+    std::collections::HashSet<String>,
-    attributes_for_faceting
+    filterable_attributes
 );
 
 make_setting_route!(
@@ -126,7 +126,7 @@ macro_rules! create_services {
 }
 
 create_services!(
-    attributes_for_faceting,
+    filterable_attributes,
     displayed_attributes,
     searchable_attributes,
     distinct_attribute,

meilisearch-http/tests/assets/dumps/v1/test/settings.json

@@ -50,7 +50,7 @@
         "wolverine": ["xmen", "logan"],
         "logan": ["wolverine", "xmen"]
     },
-    "attributesForFaceting": [
+    "filterableAttributes": [
         "gender",
         "color",
         "tags",

meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl

@@ -1,2 +1,2 @@
-{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}}
+{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","filterable_attributes":"Nothing"}}}
 {"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}}

meilisearch-http/tests/settings/get_settings.rs

@@ -19,7 +19,7 @@ async fn get_settings() {
     assert_eq!(settings.keys().len(), 6);
     assert_eq!(settings["displayedAttributes"], json!(["*"]));
     assert_eq!(settings["searchableAttributes"], json!(["*"]));
-    assert_eq!(settings["attributesForFaceting"], json!({}));
+    assert_eq!(settings["filterableAttributes"], json!([]));
     assert_eq!(settings["distinctAttribute"], json!(null));
     assert_eq!(
         settings["rankingRules"],
@@ -72,26 +72,44 @@ async fn delete_settings_unexisting_index() {
 async fn reset_all_settings() {
     let server = Server::new().await;
     let index = server.index("test");
-    index
+
-        .update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"], "stopWords": ["the"], "attributesForFaceting": { "toto": "string" } }))
+    let documents = json!([
-        .await;
+        {
+            "id": 1,
+            "name": "curqui",
+            "age": 99
+        }
+    ]);
+
+    let (response, code) = index.add_documents(documents, None).await;
+    assert_eq!(code, 202);
+    assert_eq!(response["updateId"], 0);
     index.wait_update_id(0).await;
+
+    index
+        .update_settings(json!({"displayedAttributes": ["name", "age"], "searchableAttributes": ["name"], "stopWords": ["the"], "filterableAttributes": ["age"] }))
+        .await;
+    index.wait_update_id(1).await;
     let (response, code) = index.settings().await;
     assert_eq!(code, 200);
-    assert_eq!(response["displayedAttributes"], json!(["foo"]));
+    assert_eq!(response["displayedAttributes"], json!(["name", "age"]));
-    assert_eq!(response["searchableAttributes"], json!(["bar"]));
+    assert_eq!(response["searchableAttributes"], json!(["name"]));
     assert_eq!(response["stopWords"], json!(["the"]));
-    assert_eq!(response["attributesForFaceting"], json!({"toto": "string"}));
+    assert_eq!(response["filterableAttributes"], json!(["age"]));
 
     index.delete_settings().await;
-    index.wait_update_id(1).await;
+    index.wait_update_id(2).await;
 
     let (response, code) = index.settings().await;
     assert_eq!(code, 200);
     assert_eq!(response["displayedAttributes"], json!(["*"]));
     assert_eq!(response["searchableAttributes"], json!(["*"]));
     assert_eq!(response["stopWords"], json!([]));
-    assert_eq!(response["attributesForFaceting"], json!({}));
+    assert_eq!(response["filterableAttributes"], json!([]));
+
+    let (response, code) = index.get_document(1, None).await;
+    assert_eq!(code, 200);
+    assert!(response.as_object().unwrap().get("age").is_some());
 }
 
 #[actix_rt::test]
@@ -163,7 +181,7 @@ macro_rules! test_setting_routes {
 }
 
 test_setting_routes!(
-    attributes_for_faceting,
+    filterable_attributes,
     displayed_attributes,
     searchable_attributes,
     stop_words