snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 03:55:07 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Start jayson integration

Browse Source
This commit is contained in:
Loïc Lecrenier 2022-06-21 12:16:25 +02:00
parent 9082679609
commit ba82584328
8 changed files with 283 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
Cargo.lock generated
View File

@ -1127,6 +1127,14 @@ dependencies = [
"winapi",
]
[[package]]
name = "filter-parser"
version = "0.29.3"
dependencies = [
"nom",
"nom_locate",
]
[[package]]
name = "filter-parser"
version = "0.29.3"
@ -1152,6 +1160,13 @@ dependencies = [
"miniz_oxide",
]
[[package]]
name = "flatten-serde-json"
version = "0.29.3"
dependencies = [
"serde_json",
]
[[package]]
name = "flatten-serde-json"
version = "0.29.3"
@ -1683,6 +1698,13 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "json-depth-checker"
version = "0.29.3"
dependencies = [
"serde_json",
]
[[package]]
name = "json-depth-checker"
version = "0.29.3"
@ -2031,7 +2053,7 @@ dependencies = [
"enum-iterator",
"hmac",
"meilisearch-types",
"milli",
"milli 0.29.3 (git+https://github.com/meilisearch/milli.git?tag=v0.29.3)",
"rand",
"serde",
"serde_json",
@ -2140,11 +2162,12 @@ dependencies = [
"http",
"indexmap",
"itertools",
"jayson",
"lazy_static",
"log",
"meilisearch-auth",
"meilisearch-types",
"milli",
"milli 0.29.3",
"mime",
"mockall",
"nelson",
@ -2182,6 +2205,8 @@ name = "meilisearch-types"
version = "0.28.0"
dependencies = [
"actix-web",
"jayson",
"milli 0.29.3",
"proptest",
"proptest-derive",
"serde",
@ -2212,6 +2237,51 @@ dependencies = [
"autocfg",
]
[[package]]
name = "milli"
version = "0.29.3"
dependencies = [
"bimap",
"bincode",
"bstr",
"byteorder",
"charabia",
"concat-arrays",
"crossbeam-channel",
"csv",
"either",
"filter-parser 0.29.3",
"flatten-serde-json 0.29.3",
"fst",
"fxhash",
"geoutils",
"grenad",
"heed",
"itertools",
"jayson",
"json-depth-checker 0.29.3",
"levenshtein_automata",
"log",
"logging_timer",
"memmap2",
"obkv",
"once_cell",
"ordered-float",
"rayon",
"roaring",
"rstar",
"serde",
"serde_json",
"slice-group-by",
"smallstr",
"smallvec",
"smartstring",
"tempfile",
"thiserror",
"time 0.3.9",
"uuid",
]
[[package]]
name = "milli"
version = "0.29.3"
@ -2226,15 +2296,15 @@ dependencies = [
"crossbeam-channel",
"csv",
"either",
"filter-parser",
"flatten-serde-json",
"filter-parser 0.29.3 (git+https://github.com/meilisearch/milli.git?tag=v0.29.3)",
"flatten-serde-json 0.29.3 (git+https://github.com/meilisearch/milli.git?tag=v0.29.3)",
"fst",
"fxhash",
"geoutils",
"grenad",
"heed",
"itertools",
"json-depth-checker",
"json-depth-checker 0.29.3 (git+https://github.com/meilisearch/milli.git?tag=v0.29.3)",
"levenshtein_automata",
"log",
"logging_timer",

63
meilisearch-http/src/extractors/jayson.rs
View File

@ -1,6 +1,7 @@
use actix_web::{dev::Payload, web::Json, FromRequest, HttpRequest};
use futures::ready;
use jayson::{DeserializeError, DeserializeFromValue};
use jayson::{DeserializeError, DeserializeFromValue, MergeWithError, ValuePointer};
use meilisearch_lib::milli::AscDescError;
use meilisearch_types::error::{Code, ErrorCode, ResponseError};
use std::{
fmt::Debug,
@ -10,6 +11,66 @@ use std::{
task::{Context, Poll},
};
// pub struct MeilisearchDeserializeError {
// pub Vec<(ValuePointer, Box<dyn Error>)>,
// }
// impl MergeWithError<AscDescError> for MeilisearchDeserializeError {
// fn merge(self_: Option<Self>, other: AscDescError, merge_location: jayson::ValuePointerRef) -> Result<Self, Self> {
// todo!()
// }
// }
// /*
// {
// !
// x: {
// y: {
// z: {
// a: 2
// }
// }
// }
// }
// */
// impl MergeWithError<MeilisearchDeserializeError> for MeilisearchDeserializeError {
// }
// impl DeserializeError for MeilisearchDeserializeError{
// fn location(&self) -> Option<jayson::ValuePointer> {
// todo!()
// }
// fn incorrect_value_kind(
// self_: Option<Self>,
// actual: jayson::ValueKind,
// accepted: &[jayson::ValueKind],
// location: jayson::ValuePointerRef,
// ) -> Result<Self, Self> {
// todo!()
// }
// fn missing_field(
// self_: Option<Self>,
// field: &str,
// location: jayson::ValuePointerRef,
// ) -> Result<Self, Self> {
// todo!()
// }
// fn unknown_key(
// self_: Option<Self>,
// key: &str,
// accepted: &[&str],
// location: jayson::ValuePointerRef,
// ) -> Result<Self, Self> {
// todo!()
// }
// fn unexpected(self_: Option<Self>, msg: &str, location: jayson::ValuePointerRef) -> Result<Self, Self> {
// todo!()
// }
// }
/// Extractor for typed data from Json request payloads
/// deserialised by Jayson.
///

50
meilisearch-http/src/routes/indexes/settings.rs
View File

@ -4,11 +4,13 @@ use actix_web::{web, HttpRequest, HttpResponse};
use meilisearch_lib::index::{Settings, Unchecked};
use meilisearch_lib::index_controller::Update;
use meilisearch_lib::MeiliSearch;
use meilisearch_types::error::ResponseError;
use meilisearch_types::error::{MeiliDeserError, ResponseError};
use serde_json::json;
use crate::analytics::Analytics;
use crate::error::MeilisearchHttpError;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::extractors::jayson::ValidatedJson;
use crate::task::SummarizedTaskView;
#[macro_export]
@ -260,27 +262,27 @@ make_setting_route!(
"distinctAttribute"
);
make_setting_route!(
"/ranking-rules",
put,
Vec<String>,
ranking_rules,
"rankingRules",
analytics,
|setting: &Option<Vec<String>>, req: &HttpRequest| {
use serde_json::json;
// make_setting_route!(
// "/ranking-rules",
// put,
// Vec<String>,
// ranking_rules,
// "rankingRules",
// analytics,
// |setting: &Option<Vec<milli::AscDesc>>, req: &HttpRequest| {
// use serde_json::json;
analytics.publish(
"RankingRules Updated".to_string(),
json!({
"ranking_rules": {
"sort_position": setting.as_ref().map(|sort| sort.iter().position(|s| s == "sort")),
}
}),
Some(req),
);
}
);
// analytics.publish(
// "RankingRules Updated".to_string(),
// json!({
// "ranking_rules": {
// "sort_position": setting.as_ref().map(|sort| sort.iter().position(|s| s == "sort")),
// }
// }),
// Some(req),
// );
// }
// );
make_setting_route!(
"/faceting",
@ -348,14 +350,14 @@ generate_configure!(
distinct_attribute,
stop_words,
synonyms,
ranking_rules,
// ranking_rules,
typo_tolerance
);
pub async fn update_all(
meilisearch: GuardedData<ActionPolicy<{ actions::SETTINGS_UPDATE }>, MeiliSearch>,
index_uid: web::Path<String>,
body: web::Json<Settings<Unchecked>>,
body: ValidatedJson<Settings<Unchecked>, MeiliDeserError>,
req: HttpRequest,
analytics: web::Data<dyn Analytics>,
) -> Result<HttpResponse, ResponseError> {
@ -365,7 +367,7 @@ pub async fn update_all(
"Settings Updated".to_string(),
json!({
"ranking_rules": {
"sort_position": settings.ranking_rules.as_ref().set().map(|sort| sort.iter().position(|s| s == "sort")),
"sort_position": settings.ranking_rules.as_ref().set().map(|sort| sort.iter().position(|s| true /*TODO*/)),
},
"searchable_attributes": {
"total": settings.searchable_attributes.as_ref().set().map(|searchable| searchable.len()),

3
meilisearch-lib/Cargo.toml
View File

@ -30,7 +30,8 @@ lazy_static = "1.4.0"
log = "0.4.14"
meilisearch-auth = { path = "../meilisearch-auth" }
meilisearch-types = { path = "../meilisearch-types" }
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.29.3" }
milli = { path = "../../milli/milli" }
jayson = { path = "../../jayson" }
mime = "0.3.16"
num_cpus = "1.13.1"
obkv = "0.2.0"

4
meilisearch-lib/src/index/index.rs
View File

@ -8,7 +8,7 @@ use std::sync::Arc;
use fst::IntoStreamer;
use milli::heed::{EnvOpenOptions, RoTxn};
use milli::update::{IndexerConfig, Setting};
use milli::{obkv_to_json, FieldDistribution, DEFAULT_VALUES_PER_FACET};
use milli::{obkv_to_json, AscDesc, FieldDistribution, Member, DEFAULT_VALUES_PER_FACET};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use time::OffsetDateTime;
@ -146,7 +146,7 @@ impl Index {
let criteria = self
.criteria(txn)?
.into_iter()
.map(|c| c.to_string())
.map(|c| AscDesc::Asc(Member::Field("todo".to_string())))
.collect();
let stop_words = self

45
meilisearch-lib/src/index/updates.rs
View File

@ -1,14 +1,16 @@
use std::collections::{BTreeMap, BTreeSet};
use std::marker::PhantomData;
use std::num::NonZeroUsize;
use jayson::DeserializeFromValue;
use log::{debug, info, trace};
use milli::documents::DocumentBatchReader;
use milli::update::{
DocumentAdditionResult, DocumentDeletionResult, IndexDocumentsConfig, IndexDocumentsMethod,
Setting,
};
use milli::{AscDesc, Criterion};
use rayon::vec;
use serde::{Deserialize, Serialize, Serializer};
use std::collections::{BTreeMap, BTreeSet};
use std::marker::PhantomData;
use std::num::NonZeroUsize;
use uuid::Uuid;
use super::error::Result;
@ -36,9 +38,23 @@ pub struct Checked;
#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq)]
pub struct Unchecked;
impl<E> DeserializeFromValue<E> for Unchecked
where
E: jayson::DeserializeError,
{
fn deserialize_from_value<V>(
value: jayson::Value<V>,
location: jayson::ValuePointerRef,
) -> std::result::Result<Self, E>
where
V: jayson::IntoValue,
{
Ok(Unchecked)
}
}
#[cfg_attr(test, derive(proptest_derive::Arbitrary))]
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, DeserializeFromValue)]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
pub struct MinWordSizeTyposSetting {
@ -51,7 +67,9 @@ pub struct MinWordSizeTyposSetting {
}
#[cfg_attr(test, derive(proptest_derive::Arbitrary))]
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
#[derive(
Debug, Clone, Default, Serialize, Deserialize, PartialEq, jayson::DeserializeFromValue,
)]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
pub struct TypoSettings {
@ -70,7 +88,7 @@ pub struct TypoSettings {
}
#[cfg_attr(test, derive(proptest_derive::Arbitrary))]
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, DeserializeFromValue)]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
pub struct FacetingSettings {
@ -80,7 +98,9 @@ pub struct FacetingSettings {
}
#[cfg_attr(test, derive(proptest_derive::Arbitrary))]
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
#[derive(
Debug, Clone, Default, Serialize, Deserialize, PartialEq, jayson::DeserializeFromValue,
)]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
pub struct PaginationSettings {
@ -92,11 +112,15 @@ pub struct PaginationSettings {
/// Holds all the settings for an index. `T` can either be `Checked` if they represents settings
/// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a
/// call to `check` will return a `Settings<Checked>` from a `Settings<Unchecked>`.
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
#[derive(
Debug, Clone, Default, Serialize, Deserialize, PartialEq, jayson::DeserializeFromValue,
)]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
#[serde(bound(serialize = "T: Serialize", deserialize = "T: Deserialize<'static>"))]
#[cfg_attr(test, derive(proptest_derive::Arbitrary))]
#[jayson(rename_all = camelCase, deny_unknown_fields)]
pub struct Settings<T> {
#[serde(
default,
@ -122,7 +146,8 @@ pub struct Settings<T> {
pub sortable_attributes: Setting<BTreeSet<String>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
#[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))]
pub ranking_rules: Setting<Vec<String>>,
#[jayson(needs_predicate)]
pub ranking_rules: Setting<Vec<AscDesc>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
#[cfg_attr(test, proptest(strategy = "test::setting_strategy()"))]
pub stop_words: Setting<BTreeSet<String>>,

2
meilisearch-types/Cargo.toml
View File

@ -10,6 +10,8 @@ proptest = { version = "1.0.0", optional = true }
proptest-derive = { version = "0.3.0", optional = true }
serde = { version = "1.0.136", features = ["derive"] }
serde_json = "1.0.79"
jayson = { path = "../../jayson" }
milli = { path = "../../milli/milli" }
[features]
test-traits = ["proptest", "proptest-derive"]

79
meilisearch-types/src/error.rs
View File

@ -1,6 +1,7 @@
use std::fmt;
use actix_web::{self as aweb, http::StatusCode, HttpResponseBuilder};
use jayson::{ValueKind, ValuePointerRef};
use serde::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
@ -22,6 +23,84 @@ pub struct ResponseError {
error_link: String,
}
#[derive(Debug)]
pub struct MeiliDeserError(String);
impl std::fmt::Display for MeiliDeserError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.0)
}
}
impl std::error::Error for MeiliDeserError {}
impl ErrorCode for MeiliDeserError {
fn error_code(&self) -> Code {
Code::MalformedPayload
}
}
impl jayson::MergeWithError<milli::AscDescError> for MeiliDeserError {
fn merge(
self_: Option<Self>,
other: milli::AscDescError,
merge_location: jayson::ValuePointerRef,
) -> Result<Self, Self> {
let pointer = merge_location.to_owned();
Err(MeiliDeserError(format!("{pointer:?} -> {other} ")))
}
}
impl jayson::MergeWithError<MeiliDeserError> for MeiliDeserError {
fn merge(
self_: Option<Self>,
other: MeiliDeserError,
merge_location: ValuePointerRef,
) -> Result<Self, Self> {
Err(other)
}
}
impl jayson::DeserializeError for MeiliDeserError {
/// Return the origin of the error, if it can be found
fn location(&self) -> Option<jayson::ValuePointer> {
None
}
/// Create a new error due to an unexpected value kind.
///
/// Return `Ok` to continue deserializing or `Err` to fail early.
fn incorrect_value_kind(
self_: Option<Self>,
actual: ValueKind,
accepted: &[ValueKind],
location: ValuePointerRef,
) -> Result<Self, Self> {
Err(MeiliDeserError(format!("incorrect value kind {actual}")))
}
/// Create a new error due to a missing key.
///
/// Return `Ok` to continue deserializing or `Err` to fail early.
fn missing_field(
self_: Option<Self>,
field: &str,
location: ValuePointerRef,
) -> Result<Self, Self> {
Err(MeiliDeserError(format!("missing field {field}")))
}
/// Create a new error due to finding an unknown key.
///
/// Return `Ok` to continue deserializing or `Err` to fail early.
fn unknown_key(
self_: Option<Self>,
key: &str,
accepted: &[&str],
location: ValuePointerRef,
) -> Result<Self, Self> {
Err(MeiliDeserError(format!("unknown key {key}")))
}
/// Create a new error with the custom message.
///
/// Return `Ok` to continue deserializing or `Err` to fail early.
fn unexpected(self_: Option<Self>, msg: &str, location: ValuePointerRef) -> Result<Self, Self> {
Err(MeiliDeserError(format!("unexpected {msg}")))
}
}
impl ResponseError {
pub fn from_msg(message: String, code: Code) -> Self {
Self {