From 6edef07e291d5a3ce280ffe2136ffca0ae9656e2 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Dec 2019 14:06:16 +0100 Subject: [PATCH 1/7] HTTP delete index route; Fix error on index not found --- meilisearch-http/src/routes/index.rs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/index.rs index 3df71cc76..bf37e2a28 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/index.rs @@ -390,19 +390,10 @@ pub async fn get_all_updates_status(ctx: Context) -> SResult { pub async fn delete_index(ctx: Context) -> SResult { ctx.is_allowed(IndexesWrite)?; + let _ = ctx.index()?; let index_uid = ctx.url_param("index")?; - - let found = ctx - .state() - .db - .delete_index(&index_uid) - .map_err(ResponseError::internal)?; - - if found { - Ok(StatusCode::NO_CONTENT) - } else { - Ok(StatusCode::NOT_FOUND) - } + ctx.state().db.delete_index(&index_uid).map_err(ResponseError::internal)?; + Ok(StatusCode::NO_CONTENT) } pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) { From 88b3c051551ad49e3c0ac94ea89dcc27d1d02fb8 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Dec 2019 15:31:39 +0100 Subject: [PATCH 2/7] Stop words; Do not reindex all documents if there is no documents --- .../src/update/stop_words_deletion.rs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/meilisearch-core/src/update/stop_words_deletion.rs b/meilisearch-core/src/update/stop_words_deletion.rs index 2ef438454..9c799b402 100644 --- a/meilisearch-core/src/update/stop_words_deletion.rs +++ b/meilisearch-core/src/update/stop_words_deletion.rs @@ -101,14 +101,18 @@ pub fn apply_stop_words_deletion( // now that we have setup the stop words // lets reindex everything... - reindex_all_documents( - writer, - main_store, - documents_fields_store, - documents_fields_counts_store, - postings_lists_store, - docs_words_store, - )?; + if let Ok(number) = main_store.number_of_documents(writer) { + if number > 0 { + reindex_all_documents( + writer, + main_store, + documents_fields_store, + documents_fields_counts_store, + postings_lists_store, + docs_words_store, + )?; + } + } Ok(()) } From bf80729e1721c2cb3a53e7ef793256a04b9b8db6 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Dec 2019 15:39:32 +0100 Subject: [PATCH 3/7] Update message on access forbidden --- meilisearch-http/src/error.rs | 2 +- meilisearch-http/src/helpers/tide.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index afefdec3a..6d6310cd0 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -79,7 +79,7 @@ impl IntoResponse for ResponseError { error(err, StatusCode::BAD_REQUEST) } ResponseError::InvalidToken(err) => { - error(format!("Invalid Token: {}", err), StatusCode::FORBIDDEN) + error(format!("Invalid API key: {}", err), StatusCode::FORBIDDEN) } ResponseError::NotFound(err) => error(err, StatusCode::NOT_FOUND), ResponseError::IndexNotFound(index) => { diff --git a/meilisearch-http/src/helpers/tide.rs b/meilisearch-http/src/helpers/tide.rs index 3edbc81e6..52c785fee 100644 --- a/meilisearch-http/src/helpers/tide.rs +++ b/meilisearch-http/src/helpers/tide.rs @@ -66,7 +66,7 @@ impl ContextExt for Context { } if !token_config.acl.contains(&acl) { - return Err(ResponseError::invalid_token("token do not have this ACL")); + return Err(ResponseError::invalid_token("no permission")); } Ok(()) From 04d34cb8aae2736d30f191d9569bf0b824a6cc73 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Dec 2019 16:36:42 +0100 Subject: [PATCH 4/7] Search; return formated section only if it's necessary --- meilisearch-http/src/helpers/meilisearch.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/helpers/meilisearch.rs b/meilisearch-http/src/helpers/meilisearch.rs index fc7958c49..6ce77ba91 100644 --- a/meilisearch-http/src/helpers/meilisearch.rs +++ b/meilisearch-http/src/helpers/meilisearch.rs @@ -243,7 +243,14 @@ impl<'a> SearchBuilder<'a> { .map_err(|e| Error::RetrieveDocument(doc.id.0, e.to_string()))? .ok_or(Error::DocumentNotFound(doc.id.0))?; - let mut formatted = document.clone(); + let has_attributes_to_highlight = self.attributes_to_highlight.is_some(); + let has_attributes_to_crop = self.attributes_to_crop.is_some(); + + let mut formatted = if has_attributes_to_highlight || has_attributes_to_crop { + document.clone() + } else { + IndexMap::new() + }; let mut matches = doc.highlights.clone(); // Crops fields if needed @@ -494,7 +501,7 @@ fn calculate_highlights( matches: &MatchesInfos, attributes_to_highlight: &HashSet, ) -> IndexMap { - let mut highlight_result = IndexMap::new(); + let mut highlight_result = document.clone(); for (attribute, matches) in matches.iter() { if attributes_to_highlight.contains(attribute) { From 1ea29bb92e4c7d435ec1c5cba9d5b9150147ad6b Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Dec 2019 16:37:46 +0100 Subject: [PATCH 5/7] Fix unwrap if schema does not contain ranked attributes on a custom ranking setting --- meilisearch-http/src/helpers/meilisearch.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/helpers/meilisearch.rs b/meilisearch-http/src/helpers/meilisearch.rs index 6ce77ba91..b5d772157 100644 --- a/meilisearch-http/src/helpers/meilisearch.rs +++ b/meilisearch-http/src/helpers/meilisearch.rs @@ -1,6 +1,6 @@ use crate::routes::setting::{RankingOrdering, SettingBody}; use indexmap::IndexMap; -use log::error; +use log::{error, warn}; use meilisearch_core::criterion::*; use meilisearch_core::Highlight; use meilisearch_core::{Index, RankedMap}; @@ -349,13 +349,19 @@ impl<'a> SearchBuilder<'a> { for (rule, order) in ranking_rules.iter() { let custom_ranking = match order { RankingOrdering::Asc => { - SortByAttr::lower_is_better(&ranked_map, &schema, &rule).unwrap() + SortByAttr::lower_is_better(&ranked_map, &schema, &rule) } RankingOrdering::Dsc => { - SortByAttr::higher_is_better(&ranked_map, &schema, &rule).unwrap() + SortByAttr::higher_is_better(&ranked_map, &schema, &rule) } }; - builder.push(custom_ranking); + if let Ok(custom_ranking) = custom_ranking { + builder.push(custom_ranking); + } else { + // TODO push this warning to a log tree + warn!("Custom ranking cannot be added; Attribute {} not registered for ranking", rule) + } + } builder.push(DocumentId); return Ok(Some(builder.build())); From 3a4130f34425918671f2b855210ab3e1158524e4 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Dec 2019 19:04:10 +0100 Subject: [PATCH 6/7] Allow to index files with null or boolean --- meilisearch-core/src/serde/indexer.rs | 29 ++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/meilisearch-core/src/serde/indexer.rs b/meilisearch-core/src/serde/indexer.rs index 68066069f..62ee28248 100644 --- a/meilisearch-core/src/serde/indexer.rs +++ b/meilisearch-core/src/serde/indexer.rs @@ -24,9 +24,7 @@ impl<'a> ser::Serializer for Indexer<'a> { type SerializeStructVariant = ser::Impossible; fn serialize_bool(self, _value: bool) -> Result { - Err(SerializerError::UnindexableType { - type_name: "boolean", - }) + Ok(None) } fn serialize_char(self, value: char) -> Result { @@ -96,9 +94,7 @@ impl<'a> ser::Serializer for Indexer<'a> { } fn serialize_none(self) -> Result { - Err(SerializerError::UnindexableType { - type_name: "Option", - }) + Ok(None) } fn serialize_some(self, value: &T) -> Result @@ -113,13 +109,11 @@ impl<'a> ser::Serializer for Indexer<'a> { } fn serialize_unit(self) -> Result { - Err(SerializerError::UnindexableType { type_name: "()" }) + Ok(None) } fn serialize_unit_struct(self, _name: &'static str) -> Result { - Err(SerializerError::UnindexableType { - type_name: "unit struct", - }) + Ok(None) } fn serialize_unit_variant( @@ -128,9 +122,7 @@ impl<'a> ser::Serializer for Indexer<'a> { _variant_index: u32, _variant: &'static str, ) -> Result { - Err(SerializerError::UnindexableType { - type_name: "unit variant", - }) + Ok(None) } fn serialize_newtype_struct( @@ -219,9 +211,14 @@ impl<'a> ser::Serializer for Indexer<'a> { _name: &'static str, _len: usize, ) -> Result { - Err(SerializerError::UnindexableType { - type_name: "struct", - }) + let indexer = StructIndexer { + attribute: self.attribute, + document_id: self.document_id, + indexer: self.indexer, + texts: Vec::new(), + }; + + Ok(indexer) } fn serialize_struct_variant( From 0b5db7751128a90c369dd0a6c9e7b37c8fd368c0 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 12 Dec 2019 19:10:47 +0100 Subject: [PATCH 7/7] Fix erase setting option --- meilisearch-http/src/helpers/meilisearch.rs | 4 +-- meilisearch-http/src/routes/setting.rs | 35 ++++++++++++++++----- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/meilisearch-http/src/helpers/meilisearch.rs b/meilisearch-http/src/helpers/meilisearch.rs index b5d772157..8079f7168 100644 --- a/meilisearch-http/src/helpers/meilisearch.rs +++ b/meilisearch-http/src/helpers/meilisearch.rs @@ -1,4 +1,4 @@ -use crate::routes::setting::{RankingOrdering, SettingBody}; +use crate::routes::setting::{RankingOrdering, Setting}; use indexmap::IndexMap; use log::{error, warn}; use meilisearch_core::criterion::*; @@ -299,7 +299,7 @@ impl<'a> SearchBuilder<'a> { ) -> Result>, Error> { let current_settings = match self.index.main.customs(reader).unwrap() { Some(bytes) => bincode::deserialize(bytes).unwrap(), - None => SettingBody::default(), + None => Setting::default(), }; let ranking_rules = ¤t_settings.ranking_rules; diff --git a/meilisearch-http/src/routes/setting.rs b/meilisearch-http/src/routes/setting.rs index 5793fcf47..8415aedeb 100644 --- a/meilisearch-http/src/routes/setting.rs +++ b/meilisearch-http/src/routes/setting.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use http::StatusCode; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, Deserializer}; use tide::response::IntoResponse; use tide::{Context, Response}; @@ -11,9 +11,9 @@ use crate::models::token::ACL::*; use crate::routes::document::IndexUpdateResponse; use crate::Data; -#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct SettingBody { +pub struct Setting { pub ranking_order: Option, pub distinct_field: Option, pub ranking_rules: Option, @@ -39,12 +39,31 @@ pub async fn get(ctx: Context) -> SResult { let settings = match index.main.customs(&reader).unwrap() { Some(bytes) => bincode::deserialize(bytes).unwrap(), - None => SettingBody::default(), + None => Setting::default(), }; Ok(tide::response::json(settings)) } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SettingBody { + #[serde(default, deserialize_with = "deserialize_some")] + pub ranking_order: Option>, + #[serde(default, deserialize_with = "deserialize_some")] + pub distinct_field: Option>, + #[serde(default, deserialize_with = "deserialize_some")] + pub ranking_rules: Option>, +} + +// Any value that is present is considered Some value, including null. +fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> + where T: Deserialize<'de>, + D: Deserializer<'de> +{ + Deserialize::deserialize(deserializer).map(Some) +} + pub async fn update(mut ctx: Context) -> SResult { ctx.is_allowed(SettingsWrite)?; @@ -58,19 +77,19 @@ pub async fn update(mut ctx: Context) -> SResult { let mut current_settings = match index.main.customs(&reader).unwrap() { Some(bytes) => bincode::deserialize(bytes).unwrap(), - None => SettingBody::default(), + None => Setting::default(), }; if let Some(ranking_order) = settings.ranking_order { - current_settings.ranking_order = Some(ranking_order); + current_settings.ranking_order = ranking_order; } if let Some(distinct_field) = settings.distinct_field { - current_settings.distinct_field = Some(distinct_field); + current_settings.distinct_field = distinct_field; } if let Some(ranking_rules) = settings.ranking_rules { - current_settings.ranking_rules = Some(ranking_rules); + current_settings.ranking_rules = ranking_rules; } let bytes = bincode::serialize(¤t_settings).unwrap();