From ba82584328556412b2c849c23558b3444472aa24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Tue, 21 Jun 2022 12:16:25 +0200 Subject: [PATCH] Start jayson integration --- Cargo.lock | 80 +++++++++++++++++-- meilisearch-http/src/extractors/jayson.rs | 63 ++++++++++++++- .../src/routes/indexes/settings.rs | 50 ++++++------ meilisearch-lib/Cargo.toml | 3 +- meilisearch-lib/src/index/index.rs | 4 +- meilisearch-lib/src/index/updates.rs | 45 ++++++++--- meilisearch-types/Cargo.toml | 2 + meilisearch-types/src/error.rs | 79 ++++++++++++++++++ 8 files changed, 283 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 969153813..f5bcb4eff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1127,6 +1127,14 @@ dependencies = [ "winapi", ] +[[package]] +name = "filter-parser" +version = "0.29.3" +dependencies = [ + "nom", + "nom_locate", +] + [[package]] name = "filter-parser" version = "0.29.3" @@ -1152,6 +1160,13 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flatten-serde-json" +version = "0.29.3" +dependencies = [ + "serde_json", +] + [[package]] name = "flatten-serde-json" version = "0.29.3" @@ -1683,6 +1698,13 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-depth-checker" +version = "0.29.3" +dependencies = [ + "serde_json", +] + [[package]] name = "json-depth-checker" version = "0.29.3" @@ -2031,7 +2053,7 @@ dependencies = [ "enum-iterator", "hmac", "meilisearch-types", - "milli", + "milli 0.29.3 (git+https://github.com/meilisearch/milli.git?tag=v0.29.3)", "rand", "serde", "serde_json", @@ -2140,11 +2162,12 @@ dependencies = [ "http", "indexmap", "itertools", + "jayson", "lazy_static", "log", "meilisearch-auth", "meilisearch-types", - "milli", + "milli 0.29.3", "mime", "mockall", "nelson", @@ -2182,6 +2205,8 @@ name = "meilisearch-types" version = "0.28.0" dependencies = [ "actix-web", + "jayson", + "milli 0.29.3", "proptest", "proptest-derive", "serde", @@ -2212,6 +2237,51 @@ dependencies = [ "autocfg", ] +[[package]] +name = "milli" +version = "0.29.3" +dependencies = [ + "bimap", + "bincode", + "bstr", + "byteorder", + "charabia", + "concat-arrays", + "crossbeam-channel", + "csv", + "either", + "filter-parser 0.29.3", + "flatten-serde-json 0.29.3", + "fst", + "fxhash", + "geoutils", + "grenad", + "heed", + "itertools", + "jayson", + "json-depth-checker 0.29.3", + "levenshtein_automata", + "log", + "logging_timer", + "memmap2", + "obkv", + "once_cell", + "ordered-float", + "rayon", + "roaring", + "rstar", + "serde", + "serde_json", + "slice-group-by", + "smallstr", + "smallvec", + "smartstring", + "tempfile", + "thiserror", + "time 0.3.9", + "uuid", +] + [[package]] name = "milli" version = "0.29.3" @@ -2226,15 +2296,15 @@ dependencies = [ "crossbeam-channel", "csv", "either", - "filter-parser", - "flatten-serde-json", + "filter-parser 0.29.3 (git+https://github.com/meilisearch/milli.git?tag=v0.29.3)", + "flatten-serde-json 0.29.3 (git+https://github.com/meilisearch/milli.git?tag=v0.29.3)", "fst", "fxhash", "geoutils", "grenad", "heed", "itertools", - "json-depth-checker", + "json-depth-checker 0.29.3 (git+https://github.com/meilisearch/milli.git?tag=v0.29.3)", "levenshtein_automata", "log", "logging_timer", diff --git a/meilisearch-http/src/extractors/jayson.rs b/meilisearch-http/src/extractors/jayson.rs index 3af7e4876..4cfb45138 100644 --- a/meilisearch-http/src/extractors/jayson.rs +++ b/meilisearch-http/src/extractors/jayson.rs @@ -1,6 +1,7 @@ use actix_web::{dev::Payload, web::Json, FromRequest, HttpRequest}; use futures::ready; -use jayson::{DeserializeError, DeserializeFromValue}; +use jayson::{DeserializeError, DeserializeFromValue, MergeWithError, ValuePointer}; +use meilisearch_lib::milli::AscDescError; use meilisearch_types::error::{Code, ErrorCode, ResponseError}; use std::{ fmt::Debug, @@ -10,6 +11,66 @@ use std::{ task::{Context, Poll}, }; +// pub struct MeilisearchDeserializeError { +// pub Vec<(ValuePointer, Box)>, +// } +// impl MergeWithError for MeilisearchDeserializeError { +// fn merge(self_: Option, other: AscDescError, merge_location: jayson::ValuePointerRef) -> Result { +// todo!() +// } +// } +// /* +// { +// ! +// x: { +// y: { +// z: { +// a: 2 +// } +// } +// } +// } + +// */ +// impl MergeWithError for MeilisearchDeserializeError { + +// } +// impl DeserializeError for MeilisearchDeserializeError{ +// fn location(&self) -> Option { +// todo!() +// } + +// fn incorrect_value_kind( +// self_: Option, +// actual: jayson::ValueKind, +// accepted: &[jayson::ValueKind], +// location: jayson::ValuePointerRef, +// ) -> Result { +// todo!() +// } + +// fn missing_field( +// self_: Option, +// field: &str, +// location: jayson::ValuePointerRef, +// ) -> Result { +// todo!() +// } + +// fn unknown_key( +// self_: Option, +// key: &str, +// accepted: &[&str], +// location: jayson::ValuePointerRef, +// ) -> Result { +// todo!() +// } + +// fn unexpected(self_: Option, msg: &str, location: jayson::ValuePointerRef) -> Result { +// todo!() +// } +// } + /// Extractor for typed data from Json request payloads /// deserialised by Jayson. /// diff --git a/meilisearch-http/src/routes/indexes/settings.rs b/meilisearch-http/src/routes/indexes/settings.rs index 962fe7d82..083a6bd01 100644 --- a/meilisearch-http/src/routes/indexes/settings.rs +++ b/meilisearch-http/src/routes/indexes/settings.rs @@ -4,11 +4,13 @@ use actix_web::{web, HttpRequest, HttpResponse}; use meilisearch_lib::index::{Settings, Unchecked}; use meilisearch_lib::index_controller::Update; use meilisearch_lib::MeiliSearch; -use meilisearch_types::error::ResponseError; +use meilisearch_types::error::{MeiliDeserError, ResponseError}; use serde_json::json; use crate::analytics::Analytics; +use crate::error::MeilisearchHttpError; use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::extractors::jayson::ValidatedJson; use crate::task::SummarizedTaskView; #[macro_export] @@ -260,27 +262,27 @@ make_setting_route!( "distinctAttribute" ); -make_setting_route!( - "/ranking-rules", - put, - Vec, - ranking_rules, - "rankingRules", - analytics, - |setting: &Option>, req: &HttpRequest| { - use serde_json::json; +// make_setting_route!( +// "/ranking-rules", +// put, +// Vec, +// ranking_rules, +// "rankingRules", +// analytics, +// |setting: &Option>, req: &HttpRequest| { +// use serde_json::json; - analytics.publish( - "RankingRules Updated".to_string(), - json!({ - "ranking_rules": { - "sort_position": setting.as_ref().map(|sort| sort.iter().position(|s| s == "sort")), - } - }), - Some(req), - ); - } -); +// analytics.publish( +// "RankingRules Updated".to_string(), +// json!({ +// "ranking_rules": { +// "sort_position": setting.as_ref().map(|sort| sort.iter().position(|s| s == "sort")), +// } +// }), +// Some(req), +// ); +// } +// ); make_setting_route!( "/faceting", @@ -348,14 +350,14 @@ generate_configure!( distinct_attribute, stop_words, synonyms, - ranking_rules, + // ranking_rules, typo_tolerance ); pub async fn update_all( meilisearch: GuardedData, MeiliSearch>, index_uid: web::Path, - body: web::Json>, + body: ValidatedJson, MeiliDeserError>, req: HttpRequest, analytics: web::Data, ) -> Result { @@ -365,7 +367,7 @@ pub async fn update_all( "Settings Updated".to_string(), json!({ "ranking_rules": { - "sort_position": settings.ranking_rules.as_ref().set().map(|sort| sort.iter().position(|s| s == "sort")), + "sort_position": settings.ranking_rules.as_ref().set().map(|sort| sort.iter().position(|s| true /*TODO*/)), }, "searchable_attributes": { "total": settings.searchable_attributes.as_ref().set().map(|searchable| searchable.len()), diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index 094c79901..89952995a 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -30,7 +30,8 @@ lazy_static = "1.4.0" log = "0.4.14" meilisearch-auth = { path = "../meilisearch-auth" } meilisearch-types = { path = "../meilisearch-types" } -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.29.3" } +milli = { path = "../../milli/milli" } +jayson = { path = "../../jayson" } mime = "0.3.16" num_cpus = "1.13.1" obkv = "0.2.0" diff --git a/meilisearch-lib/src/index/index.rs b/meilisearch-lib/src/index/index.rs index d4772b73b..6ceebe5e6 100644 --- a/meilisearch-lib/src/index/index.rs +++ b/meilisearch-lib/src/index/index.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use fst::IntoStreamer; use milli::heed::{EnvOpenOptions, RoTxn}; use milli::update::{IndexerConfig, Setting}; -use milli::{obkv_to_json, FieldDistribution, DEFAULT_VALUES_PER_FACET}; +use milli::{obkv_to_json, AscDesc, FieldDistribution, Member, DEFAULT_VALUES_PER_FACET}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use time::OffsetDateTime; @@ -146,7 +146,7 @@ impl Index { let criteria = self .criteria(txn)? .into_iter() - .map(|c| c.to_string()) + .map(|c| AscDesc::Asc(Member::Field("todo".to_string()))) .collect(); let stop_words = self diff --git a/meilisearch-lib/src/index/updates.rs b/meilisearch-lib/src/index/updates.rs index 95edbbf9d..2300f5c97 100644 --- a/meilisearch-lib/src/index/updates.rs +++ b/meilisearch-lib/src/index/updates.rs @@ -1,14 +1,16 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::marker::PhantomData; -use std::num::NonZeroUsize; - +use jayson::DeserializeFromValue; use log::{debug, info, trace}; use milli::documents::DocumentBatchReader; use milli::update::{ DocumentAdditionResult, DocumentDeletionResult, IndexDocumentsConfig, IndexDocumentsMethod, Setting, }; +use milli::{AscDesc, Criterion}; +use rayon::vec; use serde::{Deserialize, Serialize, Serializer}; +use std::collections::{BTreeMap, BTreeSet}; +use std::marker::PhantomData; +use std::num::NonZeroUsize; use uuid::Uuid; use super::error::Result; @@ -36,9 +38,23 @@ pub struct Checked; #[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq)] pub struct Unchecked; +impl DeserializeFromValue for Unchecked +where + E: jayson::DeserializeError, +{ + fn deserialize_from_value( + value: jayson::Value, + location: jayson::ValuePointerRef, + ) -> std::result::Result + where + V: jayson::IntoValue, + { + Ok(Unchecked) + } +} #[cfg_attr(test, derive(proptest_derive::Arbitrary))] -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, DeserializeFromValue)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct MinWordSizeTyposSetting { @@ -51,7 +67,9 @@ pub struct MinWordSizeTyposSetting { } #[cfg_attr(test, derive(proptest_derive::Arbitrary))] -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[derive( + Debug, Clone, Default, Serialize, Deserialize, PartialEq, jayson::DeserializeFromValue, +)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct TypoSettings { @@ -70,7 +88,7 @@ pub struct TypoSettings { } #[cfg_attr(test, derive(proptest_derive::Arbitrary))] -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, DeserializeFromValue)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct FacetingSettings { @@ -80,7 +98,9 @@ pub struct FacetingSettings { } #[cfg_attr(test, derive(proptest_derive::Arbitrary))] -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[derive( + Debug, Clone, Default, Serialize, Deserialize, PartialEq, jayson::DeserializeFromValue, +)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct PaginationSettings { @@ -92,11 +112,15 @@ pub struct PaginationSettings { /// Holds all the settings for an index. `T` can either be `Checked` if they represents settings /// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a /// call to `check` will return a `Settings` from a `Settings`. -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] + +#[derive( + Debug, Clone, Default, Serialize, Deserialize, PartialEq, jayson::DeserializeFromValue, +)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] #[serde(bound(serialize = "T: Serialize", deserialize = "T: Deserialize<'static>"))] #[cfg_attr(test, derive(proptest_derive::Arbitrary))] +#[jayson(rename_all = camelCase, deny_unknown_fields)] pub struct Settings { #[serde( default, @@ -122,7 +146,8 @@ pub struct Settings { pub sortable_attributes: Setting>, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] - pub ranking_rules: Setting>, + #[jayson(needs_predicate)] + pub ranking_rules: Setting>, #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))] pub stop_words: Setting>, diff --git a/meilisearch-types/Cargo.toml b/meilisearch-types/Cargo.toml index 6949722e7..2ee1e1ae3 100644 --- a/meilisearch-types/Cargo.toml +++ b/meilisearch-types/Cargo.toml @@ -10,6 +10,8 @@ proptest = { version = "1.0.0", optional = true } proptest-derive = { version = "0.3.0", optional = true } serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.79" +jayson = { path = "../../jayson" } +milli = { path = "../../milli/milli" } [features] test-traits = ["proptest", "proptest-derive"] diff --git a/meilisearch-types/src/error.rs b/meilisearch-types/src/error.rs index 56ac65f9e..c1f37edba 100644 --- a/meilisearch-types/src/error.rs +++ b/meilisearch-types/src/error.rs @@ -1,6 +1,7 @@ use std::fmt; use actix_web::{self as aweb, http::StatusCode, HttpResponseBuilder}; +use jayson::{ValueKind, ValuePointerRef}; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] @@ -22,6 +23,84 @@ pub struct ResponseError { error_link: String, } +#[derive(Debug)] +pub struct MeiliDeserError(String); +impl std::fmt::Display for MeiliDeserError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} +impl std::error::Error for MeiliDeserError {} +impl ErrorCode for MeiliDeserError { + fn error_code(&self) -> Code { + Code::MalformedPayload + } +} +impl jayson::MergeWithError for MeiliDeserError { + fn merge( + self_: Option, + other: milli::AscDescError, + merge_location: jayson::ValuePointerRef, + ) -> Result { + let pointer = merge_location.to_owned(); + Err(MeiliDeserError(format!("{pointer:?} -> {other} "))) + } +} + +impl jayson::MergeWithError for MeiliDeserError { + fn merge( + self_: Option, + other: MeiliDeserError, + merge_location: ValuePointerRef, + ) -> Result { + Err(other) + } +} +impl jayson::DeserializeError for MeiliDeserError { + /// Return the origin of the error, if it can be found + fn location(&self) -> Option { + None + } + /// Create a new error due to an unexpected value kind. + /// + /// Return `Ok` to continue deserializing or `Err` to fail early. + fn incorrect_value_kind( + self_: Option, + actual: ValueKind, + accepted: &[ValueKind], + location: ValuePointerRef, + ) -> Result { + Err(MeiliDeserError(format!("incorrect value kind {actual}"))) + } + /// Create a new error due to a missing key. + /// + /// Return `Ok` to continue deserializing or `Err` to fail early. + fn missing_field( + self_: Option, + field: &str, + location: ValuePointerRef, + ) -> Result { + Err(MeiliDeserError(format!("missing field {field}"))) + } + /// Create a new error due to finding an unknown key. + /// + /// Return `Ok` to continue deserializing or `Err` to fail early. + fn unknown_key( + self_: Option, + key: &str, + accepted: &[&str], + location: ValuePointerRef, + ) -> Result { + Err(MeiliDeserError(format!("unknown key {key}"))) + } + /// Create a new error with the custom message. + /// + /// Return `Ok` to continue deserializing or `Err` to fail early. + fn unexpected(self_: Option, msg: &str, location: ValuePointerRef) -> Result { + Err(MeiliDeserError(format!("unexpected {msg}"))) + } +} + impl ResponseError { pub fn from_msg(message: String, code: Code) -> Self { Self {