snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 18:17:39 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Merge #197

Browse Source 
197: Update milli (v0.3.1) with filterable attributes r=MarinPostma a=curquiza

Fixes #187 and #70
Also fixes #195 

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
This commit is contained in:
bors[bot] 2021-06-14 12:19:42 +00:00 committed by GitHub
commit b119bb4ab0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 64 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Cargo.lock generated
View File

@ -1660,14 +1660,13 @@ dependencies = [
[[package]]
name = "milli"
version = "0.2.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.2.1#25f75d4d03732131e6edcf20f4d126210b159d43"
version = "0.3.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.3.1#bc020317935da4ea08061b3d4518cbbd40184856"
dependencies = [
"anyhow",
"bstr",
"byteorder",
"chrono",
"crossbeam-channel",
"csv",
"either",
"flate2",

2
meilisearch-http/Cargo.toml
View File

@ -51,7 +51,7 @@ main_error = "0.1.0"
meilisearch-error = { path = "../meilisearch-error" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" }
memmap = "0.7.0"
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.2.1" }
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.3.1" }
mime = "0.3.16"
once_cell = "1.5.2"
oxidized-json-checker = "0.3.2"

7
meilisearch-http/src/index/mod.rs
View File

@ -67,7 +67,6 @@ impl Index {
let faceted_attributes = self
.faceted_fields(&txn)?
.into_iter()
.map(|(k, v)| (k, v.to_string()))
.collect();
let criteria = self
@ -83,15 +82,15 @@ impl Index {
})
.transpose()?
.unwrap_or_else(BTreeSet::new);
let distinct_attribute = self.distinct_attribute(&txn)?.map(String::from);
let distinct_field = self.distinct_field(&txn)?.map(String::from);
Ok(Settings {
displayed_attributes: Some(displayed_attributes),
searchable_attributes: Some(searchable_attributes),
attributes_for_faceting: Some(Some(faceted_attributes)),
filterable_attributes: Some(Some(faceted_attributes)),
ranking_rules: Some(Some(criteria)),
stop_words: Some(Some(stop_words)),
distinct_attribute: Some(distinct_attribute),
distinct_attribute: Some(distinct_field),
_kind: PhantomData,
})
}

16
meilisearch-http/src/index/search.rs
View File

@ -8,7 +8,7 @@ use heed::RoTxn;
use indexmap::IndexMap;
use itertools::Itertools;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
use milli::{facet::FacetValue, FacetCondition, FieldId, FieldsIdsMap, MatchingWords};
use milli::{FilterCondition, FieldId, FieldsIdsMap, MatchingWords};
use serde::{Deserialize, Serialize};
use serde_json::Value;
@ -57,7 +57,7 @@ pub struct SearchResult {
pub offset: usize,
pub processing_time_ms: u128,
#[serde(skip_serializing_if = "Option::is_none")]
pub facet_distributions: Option<BTreeMap<String, BTreeMap<FacetValue, u64>>>,
pub facet_distributions: Option<BTreeMap<String, BTreeMap<String, u64>>>,
}
impl Index {
@ -76,7 +76,7 @@ impl Index {
if let Some(ref filter) = query.filter {
if let Some(facets) = parse_facets(filter, self, &rtxn)? {
search.facet_condition(facets);
search.filter(facets);
}
}
@ -272,7 +272,7 @@ impl Matcher for HashSet<String> {
impl Matcher for MatchingWords {
fn matches(&self, w: &str) -> bool {
self.matches(w)
self.matching_bytes(w).is_some()
}
}
@ -335,9 +335,9 @@ fn parse_facets(
facets: &Value,
index: &Index,
txn: &RoTxn,
) -> anyhow::Result<Option<FacetCondition>> {
) -> anyhow::Result<Option<FilterCondition>> {
match facets {
Value::String(expr) => Ok(Some(FacetCondition::from_str(txn, index, expr)?)),
Value::String(expr) => Ok(Some(FilterCondition::from_str(txn, index, expr)?)),
Value::Array(arr) => parse_facets_array(txn, index, arr),
v => bail!("Invalid facet expression, expected Array, found: {:?}", v),
}
@ -347,7 +347,7 @@ fn parse_facets_array(
txn: &RoTxn,
index: &Index,
arr: &[Value],
) -> anyhow::Result<Option<FacetCondition>> {
) -> anyhow::Result<Option<FilterCondition>> {
let mut ands = Vec::new();
for value in arr {
match value {
@ -369,7 +369,7 @@ fn parse_facets_array(
}
}
FacetCondition::from_array(txn, &index.0, ands)
FilterCondition::from_array(txn, &index.0, ands)
}
#[cfg(test)]

26
meilisearch-http/src/index/updates.rs
View File

@ -1,4 +1,4 @@
use std::collections::{BTreeSet, HashMap};
use std::collections::{BTreeSet, HashSet};
use std::io;
use std::marker::PhantomData;
use std::num::NonZeroUsize;
@ -51,7 +51,7 @@ pub struct Settings<T> {
deserialize_with = "deserialize_some",
skip_serializing_if = "Option::is_none"
)]
pub attributes_for_faceting: Option<Option<HashMap<String, String>>>,
pub filterable_attributes: Option<Option<HashSet<String>>>,
#[serde(
default,
@ -81,7 +81,7 @@ impl Settings<Checked> {
Settings {
displayed_attributes: Some(None),
searchable_attributes: Some(None),
attributes_for_faceting: Some(None),
filterable_attributes: Some(None),
ranking_rules: Some(None),
stop_words: Some(None),
distinct_attribute: Some(None),
@ -93,7 +93,7 @@ impl Settings<Checked> {
let Self {
displayed_attributes,
searchable_attributes,
attributes_for_faceting,
filterable_attributes,
ranking_rules,
stop_words,
distinct_attribute,
@ -103,7 +103,7 @@ impl Settings<Checked> {
Settings {
displayed_attributes,
searchable_attributes,
attributes_for_faceting,
filterable_attributes,
ranking_rules,
stop_words,
distinct_attribute,
@ -139,7 +139,7 @@ impl Settings<Unchecked> {
Settings {
displayed_attributes,
searchable_attributes,
attributes_for_faceting: self.attributes_for_faceting,
filterable_attributes: self.filterable_attributes,
ranking_rules: self.ranking_rules,
stop_words: self.stop_words,
distinct_attribute: self.distinct_attribute,
@ -252,9 +252,9 @@ impl Index {
}
}
if let Some(ref facet_types) = settings.attributes_for_faceting {
let facet_types = facet_types.clone().unwrap_or_else(HashMap::new);
builder.set_faceted_fields(facet_types);
if let Some(ref facet_types) = settings.filterable_attributes {
let facet_types = facet_types.clone().unwrap_or_else(HashSet::new);
builder.set_filterable_fields(facet_types);
}
if let Some(ref criteria) = settings.ranking_rules {
@ -273,8 +273,8 @@ impl Index {
if let Some(ref distinct_attribute) = settings.distinct_attribute {
match distinct_attribute {
Some(attr) => builder.set_distinct_attribute(attr.clone()),
None => builder.reset_distinct_attribute(),
Some(attr) => builder.set_distinct_field(attr.clone()),
None => builder.reset_distinct_field(),
}
}
@ -329,7 +329,7 @@ mod test {
let settings = Settings {
displayed_attributes: Some(Some(vec![String::from("hello")])),
searchable_attributes: Some(Some(vec![String::from("hello")])),
attributes_for_faceting: None,
filterable_attributes: None,
ranking_rules: None,
stop_words: None,
distinct_attribute: None,
@ -348,7 +348,7 @@ mod test {
let settings = Settings {
displayed_attributes: Some(Some(vec![String::from("*")])),
searchable_attributes: Some(Some(vec![String::from("hello"), String::from("*")])),
attributes_for_faceting: None,
filterable_attributes: None,
ranking_rules: None,
stop_words: None,
distinct_attribute: None,

4
meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs
View File

@ -73,7 +73,7 @@ struct Settings {
#[serde(default, deserialize_with = "deserialize_some")]
pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
#[serde(default, deserialize_with = "deserialize_some")]
pub attributes_for_faceting: Option<Option<Vec<String>>>,
pub filterable_attributes: Option<Option<Vec<String>>>,
}
fn load_index(
@ -145,7 +145,7 @@ impl From<Settings> for index_controller::Settings<Unchecked> {
// representing the name of the faceted field + the type of the field. Since the type
// was not known in the V1 of the dump we are just going to assume everything is a
// String
attributes_for_faceting: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().map(|key| (key, String::from("string"))).collect())),
filterable_attributes: settings.filterable_attributes.map(|o| o.map(|vec| vec.into_iter().collect())),
// we need to convert the old `Vec<String>` into a `BTreeSet<String>`
ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| {
match criterion.as_str() {

8
meilisearch-http/src/routes/settings/mod.rs
View File

@ -74,9 +74,9 @@ macro_rules! make_setting_route {
}
make_setting_route!(
"/indexes/{index_uid}/settings/attributes-for-faceting",
std::collections::HashMap<String, String>,
attributes_for_faceting
"/indexes/{index_uid}/settings/filterable-attributes",
std::collections::HashSet<String>,
filterable_attributes
);
make_setting_route!(
@ -126,7 +126,7 @@ macro_rules! create_services {
}
create_services!(
attributes_for_faceting,
filterable_attributes,
displayed_attributes,
searchable_attributes,
distinct_attribute,

2
meilisearch-http/tests/assets/dumps/v1/test/settings.json
View File

@ -50,7 +50,7 @@
"wolverine": ["xmen", "logan"],
"logan": ["wolverine", "xmen"]
},
"attributesForFaceting": [
"filterableAttributes": [
"gender",
"color",
"tags",

4
meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl
View File

@ -1,2 +1,2 @@
{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}}
{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}}
{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","filterable_attributes":"Nothing"}}}
{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}}

38
meilisearch-http/tests/settings/get_settings.rs
View File

@ -19,7 +19,7 @@ async fn get_settings() {
assert_eq!(settings.keys().len(), 6);
assert_eq!(settings["displayedAttributes"], json!(["*"]));
assert_eq!(settings["searchableAttributes"], json!(["*"]));
assert_eq!(settings["attributesForFaceting"], json!({}));
assert_eq!(settings["filterableAttributes"], json!([]));
assert_eq!(settings["distinctAttribute"], json!(null));
assert_eq!(
settings["rankingRules"],
@ -72,26 +72,44 @@ async fn delete_settings_unexisting_index() {
async fn reset_all_settings() {
let server = Server::new().await;
let index = server.index("test");
index
.update_settings(json!({"displayedAttributes": ["foo"], "searchableAttributes": ["bar"], "stopWords": ["the"], "attributesForFaceting": { "toto": "string" } }))
.await;
let documents = json!([
{
"id": 1,
"name": "curqui",
"age": 99
}
]);
let (response, code) = index.add_documents(documents, None).await;
assert_eq!(code, 202);
assert_eq!(response["updateId"], 0);
index.wait_update_id(0).await;
index
.update_settings(json!({"displayedAttributes": ["name", "age"], "searchableAttributes": ["name"], "stopWords": ["the"], "filterableAttributes": ["age"] }))
.await;
index.wait_update_id(1).await;
let (response, code) = index.settings().await;
assert_eq!(code, 200);
assert_eq!(response["displayedAttributes"], json!(["foo"]));
assert_eq!(response["searchableAttributes"], json!(["bar"]));
assert_eq!(response["displayedAttributes"], json!(["name", "age"]));
assert_eq!(response["searchableAttributes"], json!(["name"]));
assert_eq!(response["stopWords"], json!(["the"]));
assert_eq!(response["attributesForFaceting"], json!({"toto": "string"}));
assert_eq!(response["filterableAttributes"], json!(["age"]));
index.delete_settings().await;
index.wait_update_id(1).await;
index.wait_update_id(2).await;
let (response, code) = index.settings().await;
assert_eq!(code, 200);
assert_eq!(response["displayedAttributes"], json!(["*"]));
assert_eq!(response["searchableAttributes"], json!(["*"]));
assert_eq!(response["stopWords"], json!([]));
assert_eq!(response["attributesForFaceting"], json!({}));
assert_eq!(response["filterableAttributes"], json!([]));
let (response, code) = index.get_document(1, None).await;
assert_eq!(code, 200);
assert!(response.as_object().unwrap().get("age").is_some());
}
#[actix_rt::test]
@ -163,7 +181,7 @@ macro_rules! test_setting_routes {
}
test_setting_routes!(
attributes_for_faceting,
filterable_attributes,
displayed_attributes,
searchable_attributes,
stop_words