From c11c909bad8ecca6831d2f812e7bb96047be3f78 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 1 Jul 2021 12:02:22 +0200 Subject: [PATCH 01/53] update bors --- bors.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bors.toml b/bors.toml index e3348c36d..e19665bab 100644 --- a/bors.toml +++ b/bors.toml @@ -1,7 +1,7 @@ status = [ 'Tests on ubuntu-18.04', 'Tests on macos-latest', - 'Cargo check on Windows', + 'Tests on windows-latest ', 'Run Clippy', 'Run Rustfmt' ] From 490836a7b3bfdb38734739fd097fe23cc236ba79 Mon Sep 17 00:00:00 2001 From: Irevoire Date: Thu, 1 Jul 2021 14:41:53 +0200 Subject: [PATCH 02/53] ignore the snapshots and dumps in the gitignore (#1449) --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 3ae73d6d8..8aa76ff15 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ /*.mdb /query-history.txt /data.ms +/snapshots +/dumps From 4c6655f68cc6121c3a45d6efea8ac60c8e02bc9a Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 1 Jul 2021 11:25:33 +0200 Subject: [PATCH 03/53] ci: enable tests on windows --- .github/workflows/rust.yml | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5a8403d6d..6e3d939e2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-18.04, macos-latest] + os: [ubuntu-18.04, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 - name: Run cargo check without any default features @@ -33,22 +33,6 @@ jobs: command: test args: --locked --release - # We don't run test on Windows since we get the following error: There is not enough space on the disk. - check-on-windows: - name: Cargo check on Windows - runs-on: windows-latest - steps: - - uses: actions/checkout@v2 - - name: Run cargo check without any default features - uses: actions-rs/cargo@v1 - with: - command: check - args: --no-default-features - - name: Run cargo check with all default features - uses: actions-rs/cargo@v1 - with: - command: check - clippy: name: Run Clippy runs-on: ubuntu-18.04 From cb71b714d71b025f0b08b9408191c22cb6700798 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 1 Jul 2021 14:43:10 +0200 Subject: [PATCH 04/53] fix bors --- bors.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bors.toml b/bors.toml index e19665bab..d85949f13 100644 --- a/bors.toml +++ b/bors.toml @@ -1,7 +1,7 @@ status = [ 'Tests on ubuntu-18.04', 'Tests on macos-latest', - 'Tests on windows-latest ', + 'Tests on windows-latest', 'Run Clippy', 'Run Rustfmt' ] From acfe31151ebfbd6a7d8e86eaf2878d9652d0a16e Mon Sep 17 00:00:00 2001 From: many Date: Thu, 1 Jul 2021 14:49:22 +0200 Subject: [PATCH 05/53] Hotfix panic for unicode characters When the highlight bound is in the middle of a character or if we are out of bounds, we highlight the complete matching word. note: we should enhance the tokenizer and the Highlighter to match char indices. Fix #1368 --- meilisearch-http/src/index/search.rs | 87 ++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 2d8095559..55fa75938 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -580,13 +580,23 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { // Matcher::match since the call is expensive. if format_options.highlight && token.is_word() { if let Some(length) = matcher.matches(token.text()) { - if format_options.highlight { - out.push_str(&self.marks.0); - out.push_str(&word[..length]); - out.push_str(&self.marks.1); - out.push_str(&word[length..]); - return out; + match word.get(..length).zip(word.get(length..)) { + Some((head, tail)) => { + out.push_str(&self.marks.0); + out.push_str(head); + out.push_str(&self.marks.1); + out.push_str(tail); + } + // if we are in the middle of a character + // or if all the word should be highlighted, + // we highlight the complete word. + None => { + out.push_str(&self.marks.0); + out.push_str(&word); + out.push_str(&self.marks.1); + } } + return out; } } out.push_str(word); @@ -741,6 +751,71 @@ mod test { assert_eq!(value["author"], "J. R. R. Tolkien"); } + /// https://github.com/meilisearch/MeiliSearch/issues/1368 + #[test] + fn formatted_with_highlight_in_unicode_word() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + title, + Value::String("Go💼od luck.".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("JacobLey".into()).to_string().as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: None, + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + // emojis are deunicoded during tokenization + // TODO Tokenizer should remove spaces after deunicode + matching_words.insert("gobriefcase od", Some(11)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "Go💼od luck."); + assert_eq!(value["author"], "JacobLey"); + } + #[test] fn formatted_with_crop_2() { let stop_words = fst::Set::default(); From fa5f8f95315d7377e70323e5cc35afc905a53979 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Thu, 1 Jul 2021 14:51:44 +0200 Subject: [PATCH 06/53] Fix an issue with the update loop falsely breaking --- .../src/index_controller/update_actor/store/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index f4abeae09..34e364497 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -162,8 +162,7 @@ impl UpdateStore { // function returns a Result and we must just unlock the loop on Result. 'outer: while timeout(duration, notification_receiver.recv()) .await - .transpose() - .map_or(false, |r| r.is_ok()) + .map_or(true, |o| o.is_some()) { loop { match update_store_weak.upgrade() { From 38161ede331e857ab0bb0f10920a51b821df8476 Mon Sep 17 00:00:00 2001 From: many Date: Thu, 1 Jul 2021 16:44:17 +0200 Subject: [PATCH 07/53] Add test with special characters --- meilisearch-http/src/index/search.rs | 63 +++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 55fa75938..537dffc0f 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -753,7 +753,7 @@ mod test { /// https://github.com/meilisearch/MeiliSearch/issues/1368 #[test] - fn formatted_with_highlight_in_unicode_word() { + fn formatted_with_highlight_emoji() { let stop_words = fst::Set::default(); let mut config = AnalyzerConfig::default(); config.stop_words(&stop_words); @@ -816,6 +816,67 @@ mod test { assert_eq!(value["author"], "JacobLey"); } + #[test] + fn formatted_with_highlight_in_unicode_word() { + let stop_words = fst::Set::default(); + let mut config = AnalyzerConfig::default(); + config.stop_words(&stop_words); + let analyzer = Analyzer::new(config); + let formatter = Formatter::new(&analyzer, (String::from(""), String::from(""))); + + let mut fields = FieldsIdsMap::new(); + let title = fields.insert("title").unwrap(); + let author = fields.insert("author").unwrap(); + + let mut buf = Vec::new(); + let mut obkv = obkv::KvWriter::new(&mut buf); + obkv.insert(title, Value::String("étoile".into()).to_string().as_bytes()) + .unwrap(); + obkv.finish().unwrap(); + obkv = obkv::KvWriter::new(&mut buf); + obkv.insert( + author, + Value::String("J. R. R. Tolkien".into()) + .to_string() + .as_bytes(), + ) + .unwrap(); + obkv.finish().unwrap(); + + let obkv = obkv::KvReader::new(&buf); + + let mut formatted_options = BTreeMap::new(); + formatted_options.insert( + title, + FormatOptions { + highlight: true, + crop: None, + }, + ); + formatted_options.insert( + author, + FormatOptions { + highlight: false, + crop: None, + }, + ); + + let mut matching_words = BTreeMap::new(); + matching_words.insert("etoile", Some(1)); + + let value = format_fields( + &fields, + obkv, + &formatter, + &matching_words, + &formatted_options, + ) + .unwrap(); + + assert_eq!(value["title"], "étoile"); + assert_eq!(value["author"], "J. R. R. Tolkien"); + } + #[test] fn formatted_with_crop_2() { let stop_words = fst::Set::default(); From 9949a2a930ee8f300e755d433e7703519f2df309 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 5 Jul 2021 12:19:27 +0200 Subject: [PATCH 08/53] bump milli to 0.7.2 --- Cargo.lock | 24 ++++-------------------- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c131c13d4..0cc33b2e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1578,7 +1578,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer 0.2.3", + "meilisearch-tokenizer", "memmap", "milli", "mime", @@ -1617,22 +1617,6 @@ dependencies = [ "zip", ] -[[package]] -name = "meilisearch-tokenizer" -version = "0.2.2" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.2#eda4ed4968c8ac973cf1707ef89bd7012bb2722f" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang", -] - [[package]] name = "meilisearch-tokenizer" version = "0.2.3" @@ -1676,8 +1660,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.7.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.7.1#b4dcdbf00d232f9bc744a4a6ba4ebaf5cb592b56" +version = "0.7.2" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.7.2#007fec21fcb3accad807aa2f2cf66c168afb043f" dependencies = [ "bstr", "byteorder", @@ -1695,7 +1679,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer 0.2.2", + "meilisearch-tokenizer", "memmap", "obkv", "once_cell", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index bad303643..aa0f2a974 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.1" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.2" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" From bc417726fce2d1dba9095d30410481ab223ee22d Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 5 Jul 2021 09:43:48 +0200 Subject: [PATCH 09/53] fix update loop bug --- .../index_controller/update_actor/store/mod.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 34e364497..5949ba5fc 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -470,13 +470,6 @@ impl UpdateStore { txn.commit()?; - uuids_to_remove - .iter() - .map(|uuid| update_uuid_to_file_path(&self.path, *uuid)) - .for_each(|path| { - let _ = remove_file(path); - }); - // If the currently processing update is from our index, we wait until it is // finished before returning. This ensure that no write to the index occurs after we delete it. if let State::Processing(uuid, _) = *self.state.read() { @@ -486,6 +479,17 @@ impl UpdateStore { } } + // Finally, remove any outstanding update files. This must be done after waiting for the + // last update to ensure that the update files are not deleted before the update needs + // them. + uuids_to_remove + .iter() + .map(|uuid| update_uuid_to_file_path(&self.path, *uuid)) + .for_each(|path| { + let _ = remove_file(path); + }); + + Ok(()) } From bc85d3007642aa50d305d27f7b59fb6516c10041 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 5 Jul 2021 10:02:49 +0200 Subject: [PATCH 10/53] add test --- .../index_controller/update_actor/store/mod.rs | 1 - meilisearch-http/tests/index/delete_index.rs | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 5949ba5fc..99593cac8 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -489,7 +489,6 @@ impl UpdateStore { let _ = remove_file(path); }); - Ok(()) } diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index b0f067b24..6ce9587fc 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -1,3 +1,5 @@ +use serde_json::json; + use crate::common::Server; #[actix_rt::test] @@ -23,3 +25,16 @@ async fn delete_unexisting_index() { assert_eq!(code, 404); } + +#[actix_rt::test] +async fn loop_delete_add_documents() { + let server = Server::new().await; + let index = server.index("test"); + let documents = json!([{"id": 1, "field1": "hello"}]); + for _ in 0..50 { + let (response, code) = index.add_documents(documents.clone(), None).await; + assert_eq!(code, 202, "{}", response); + let (response, code) = index.delete().await; + assert_eq!(code, 204, "{}", response); + } +} From 575ec2a06f72f273c3a7e2c3ff583328f9c99254 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 5 Jul 2021 14:29:20 +0200 Subject: [PATCH 11/53] refactor routes --- meilisearch-http/src/lib.rs | 11 +-- meilisearch-http/src/routes/dump.rs | 6 +- meilisearch-http/src/routes/health.rs | 11 --- .../{document.rs => indexes/documents.rs} | 29 +++--- .../src/routes/{index.rs => indexes/mod.rs} | 96 ++++++++----------- .../src/routes/{ => indexes}/search.rs | 4 +- .../src/routes/{ => indexes}/settings.rs | 35 +++---- .../src/routes/indexes/updates.rs | 64 +++++++++++++ meilisearch-http/src/routes/key.rs | 23 ----- meilisearch-http/src/routes/mod.rs | 75 +++++++++++++-- meilisearch-http/src/routes/stats.rs | 56 ----------- 11 files changed, 203 insertions(+), 207 deletions(-) delete mode 100644 meilisearch-http/src/routes/health.rs rename meilisearch-http/src/routes/{document.rs => indexes/documents.rs} (88%) rename meilisearch-http/src/routes/{index.rs => indexes/mod.rs} (64%) rename meilisearch-http/src/routes/{ => indexes}/search.rs (97%) rename meilisearch-http/src/routes/{ => indexes}/settings.rs (85%) create mode 100644 meilisearch-http/src/routes/indexes/updates.rs delete mode 100644 meilisearch-http/src/routes/key.rs delete mode 100644 meilisearch-http/src/routes/stats.rs diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 0eb61f84c..bbc7cefab 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -106,20 +106,13 @@ macro_rules! create_app { use actix_web::middleware::TrailingSlash; use actix_web::App; use actix_web::{middleware, web}; - use meilisearch_http::routes::*; + use meilisearch_http::routes; use meilisearch_http::{configure_auth, configure_data, dashboard}; App::new() .configure(|s| configure_data(s, $data.clone())) .configure(|s| configure_auth(s, &$data)) - .configure(document::services) - .configure(index::services) - .configure(search::services) - .configure(settings::services) - .configure(health::services) - .configure(stats::services) - .configure(key::services) - .configure(dump::services) + .configure(routes::configure) .configure(|s| dashboard(s, $enable_frontend)) .wrap( Cors::default() diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index 1f987a588..18b143fa3 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -6,9 +6,9 @@ use crate::error::ResponseError; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::Data; -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("/dumps").route(web::post().to(create_dump))) - .service(web::resource("/dumps/{dump_uid}/status").route(web::get().to(get_dump_status))); +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service(web::resource("").route(web::post().to(create_dump))) + .service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status))); } async fn create_dump(data: GuardedData) -> Result { diff --git a/meilisearch-http/src/routes/health.rs b/meilisearch-http/src/routes/health.rs deleted file mode 100644 index 54237de1a..000000000 --- a/meilisearch-http/src/routes/health.rs +++ /dev/null @@ -1,11 +0,0 @@ -use actix_web::{web, HttpResponse}; - -use crate::error::ResponseError; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("/health").route(web::get().to(get_health))); -} - -async fn get_health() -> Result { - Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) -} diff --git a/meilisearch-http/src/routes/document.rs b/meilisearch-http/src/routes/indexes/documents.rs similarity index 88% rename from meilisearch-http/src/routes/document.rs rename to meilisearch-http/src/routes/indexes/documents.rs index 112467f67..a62a85f1b 100644 --- a/meilisearch-http/src/routes/document.rs +++ b/meilisearch-http/src/routes/indexes/documents.rs @@ -54,23 +54,20 @@ struct DocumentParam { document_id: String, } -pub fn services(cfg: &mut web::ServiceConfig) { +pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( - web::scope("/indexes/{index_uid}/documents") - .service( - web::resource("") - .route(web::get().to(get_all_documents)) - .route(web::post().guard(guard_json).to(add_documents)) - .route(web::put().guard(guard_json).to(update_documents)) - .route(web::delete().to(clear_all_documents)), - ) - // this route needs to be before the /documents/{document_id} to match properly - .service(web::resource("/delete-batch").route(web::post().to(delete_documents))) - .service( - web::resource("/{document_id}") - .route(web::get().to(get_document)) - .route(web::delete().to(delete_document)), - ), + web::resource("") + .route(web::get().to(get_all_documents)) + .route(web::post().guard(guard_json).to(add_documents)) + .route(web::put().guard(guard_json).to(update_documents)) + .route(web::delete().to(clear_all_documents)), + ) + // this route needs to be before the /documents/{document_id} to match properly + .service(web::resource("/delete-batch").route(web::post().to(delete_documents))) + .service( + web::resource("/{document_id}") + .route(web::get().to(get_document)) + .route(web::delete().to(delete_document)), ); } diff --git a/meilisearch-http/src/routes/index.rs b/meilisearch-http/src/routes/indexes/mod.rs similarity index 64% rename from meilisearch-http/src/routes/index.rs rename to meilisearch-http/src/routes/indexes/mod.rs index badbdcc10..89fa366cb 100644 --- a/meilisearch-http/src/routes/index.rs +++ b/meilisearch-http/src/routes/indexes/mod.rs @@ -3,32 +3,44 @@ use chrono::{DateTime, Utc}; use log::debug; use serde::{Deserialize, Serialize}; -use super::{IndexParam, UpdateStatusResponse}; use crate::error::ResponseError; use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::routes::IndexParam; use crate::Data; -pub fn services(cfg: &mut web::ServiceConfig) { +mod documents; +mod search; +mod settings; +mod updates; + +pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( - web::resource("indexes") + web::resource("") .route(web::get().to(list_indexes)) .route(web::post().to(create_index)), ) .service( - web::resource("/indexes/{index_uid}") - .route(web::get().to(get_index)) - .route(web::put().to(update_index)) - .route(web::delete().to(delete_index)), - ) - .service( - web::resource("/indexes/{index_uid}/updates").route(web::get().to(get_all_updates_status)), - ) - .service( - web::resource("/indexes/{index_uid}/updates/{update_id}") - .route(web::get().to(get_update_status)), + web::scope("/{index_uid}") + .service( + web::resource("") + .route(web::get().to(get_index)) + .route(web::put().to(update_index)) + .route(web::delete().to(delete_index)), + ) + .service(web::resource("/stats").route(web::get().to(get_index_stats))) + .service(web::scope("/documents").configure(documents::configure)) + .service(web::scope("/search").configure(search::configure)) + .service(web::scope("/updates").configure(updates::configure)) + .service(web::scope("/settings").configure(settings::configure)), ); } +async fn list_indexes(data: GuardedData) -> Result { + let indexes = data.list_indexes().await?; + debug!("returns: {:?}", indexes); + Ok(HttpResponse::Ok().json(indexes)) +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct IndexCreateRequest { @@ -36,6 +48,15 @@ struct IndexCreateRequest { primary_key: Option, } +async fn create_index( + data: GuardedData, + body: web::Json, +) -> Result { + let body = body.into_inner(); + let meta = data.create_index(body.uid, body.primary_key).await?; + Ok(HttpResponse::Ok().json(meta)) +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct UpdateIndexRequest { @@ -52,22 +73,6 @@ pub struct UpdateIndexResponse { updated_at: DateTime, primary_key: Option, } - -async fn list_indexes(data: GuardedData) -> Result { - let indexes = data.list_indexes().await?; - debug!("returns: {:?}", indexes); - Ok(HttpResponse::Ok().json(indexes)) -} - -async fn create_index( - data: GuardedData, - body: web::Json, -) -> Result { - let body = body.into_inner(); - let meta = data.create_index(body.uid, body.primary_key).await?; - Ok(HttpResponse::Ok().json(meta)) -} - async fn get_index( data: GuardedData, path: web::Path, @@ -99,35 +104,12 @@ async fn delete_index( Ok(HttpResponse::NoContent().finish()) } -#[derive(Deserialize)] -struct UpdateParam { - index_uid: String, - update_id: u64, -} - -async fn get_update_status( - data: GuardedData, - path: web::Path, -) -> Result { - let params = path.into_inner(); - let meta = data - .get_update_status(params.index_uid, params.update_id) - .await?; - let meta = UpdateStatusResponse::from(meta); - debug!("returns: {:?}", meta); - Ok(HttpResponse::Ok().json(meta)) -} - -async fn get_all_updates_status( +async fn get_index_stats( data: GuardedData, path: web::Path, ) -> Result { - let metas = data.get_updates_status(path.into_inner().index_uid).await?; - let metas = metas - .into_iter() - .map(UpdateStatusResponse::from) - .collect::>(); + let response = data.get_index_stats(path.index_uid.clone()).await?; - debug!("returns: {:?}", metas); - Ok(HttpResponse::Ok().json(metas)) + debug!("returns: {:?}", response); + Ok(HttpResponse::Ok().json(response)) } diff --git a/meilisearch-http/src/routes/search.rs b/meilisearch-http/src/routes/indexes/search.rs similarity index 97% rename from meilisearch-http/src/routes/search.rs rename to meilisearch-http/src/routes/indexes/search.rs index 31a7dbd03..b6820018e 100644 --- a/meilisearch-http/src/routes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -11,9 +11,9 @@ use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT}; use crate::routes::IndexParam; use crate::Data; -pub fn services(cfg: &mut web::ServiceConfig) { +pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( - web::resource("/indexes/{index_uid}/search") + web::resource("") .route(web::get().to(search_with_url_query)) .route(web::post().to(search_with_post)), ); diff --git a/meilisearch-http/src/routes/settings.rs b/meilisearch-http/src/routes/indexes/settings.rs similarity index 85% rename from meilisearch-http/src/routes/settings.rs rename to meilisearch-http/src/routes/indexes/settings.rs index 812e37b58..272060bf5 100644 --- a/meilisearch-http/src/routes/settings.rs +++ b/meilisearch-http/src/routes/indexes/settings.rs @@ -69,68 +69,63 @@ macro_rules! make_setting_route { } make_setting_route!( - "/indexes/{index_uid}/settings/filterable-attributes", + "/filterable-attributes", std::collections::HashSet, filterable_attributes, "filterableAttributes" ); make_setting_route!( - "/indexes/{index_uid}/settings/displayed-attributes", + "/displayed-attributes", Vec, displayed_attributes, "displayedAttributes" ); make_setting_route!( - "/indexes/{index_uid}/settings/searchable-attributes", + "/searchable-attributes", Vec, searchable_attributes, "searchableAttributes" ); make_setting_route!( - "/indexes/{index_uid}/settings/stop-words", + "/stop-words", std::collections::BTreeSet, stop_words, "stopWords" ); make_setting_route!( - "/indexes/{index_uid}/settings/synonyms", + "/synonyms", std::collections::BTreeMap>, synonyms, "synonyms" ); make_setting_route!( - "/indexes/{index_uid}/settings/distinct-attribute", + "/distinct-attribute", String, distinct_attribute, "distinctAttribute" ); -make_setting_route!( - "/indexes/{index_uid}/settings/ranking-rules", - Vec, - ranking_rules, - "rankingRules" -); +make_setting_route!("/ranking-rules", Vec, ranking_rules, "rankingRules"); -macro_rules! create_services { +macro_rules! generate_configure { ($($mod:ident),*) => { - pub fn services(cfg: &mut web::ServiceConfig) { - cfg - .service(web::resource("/indexes/{index_uid}/settings") - .route(web::post().to(update_all)) - .route(web::get().to(get_all)) - .route(web::delete().to(delete_all))) + pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service( + web::resource("") + .route(web::post().to(update_all)) + .route(web::get().to(get_all)) + .route(web::delete().to(delete_all))) $(.service($mod::resources()))*; } }; } -create_services!( +generate_configure!( filterable_attributes, displayed_attributes, searchable_attributes, diff --git a/meilisearch-http/src/routes/indexes/updates.rs b/meilisearch-http/src/routes/indexes/updates.rs new file mode 100644 index 000000000..735736e97 --- /dev/null +++ b/meilisearch-http/src/routes/indexes/updates.rs @@ -0,0 +1,64 @@ +use actix_web::{web, HttpResponse}; +use chrono::{DateTime, Utc}; +use log::debug; +use serde::{Deserialize, Serialize}; + +use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::routes::{IndexParam, UpdateStatusResponse}; +use crate::Data; + +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service(web::resource("").route(web::get().to(get_all_updates_status))) + .service(web::resource("{update_id}").route(web::get().to(get_update_status))); +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct UpdateIndexRequest { + uid: Option, + primary_key: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UpdateIndexResponse { + name: String, + uid: String, + created_at: DateTime, + updated_at: DateTime, + primary_key: Option, +} + +#[derive(Deserialize)] +struct UpdateParam { + index_uid: String, + update_id: u64, +} + +async fn get_update_status( + data: GuardedData, + path: web::Path, +) -> Result { + let params = path.into_inner(); + let meta = data + .get_update_status(params.index_uid, params.update_id) + .await?; + let meta = UpdateStatusResponse::from(meta); + debug!("returns: {:?}", meta); + Ok(HttpResponse::Ok().json(meta)) +} + +async fn get_all_updates_status( + data: GuardedData, + path: web::Path, +) -> Result { + let metas = data.get_updates_status(path.into_inner().index_uid).await?; + let metas = metas + .into_iter() + .map(UpdateStatusResponse::from) + .collect::>(); + + debug!("returns: {:?}", metas); + Ok(HttpResponse::Ok().json(metas)) +} diff --git a/meilisearch-http/src/routes/key.rs b/meilisearch-http/src/routes/key.rs deleted file mode 100644 index d47e264be..000000000 --- a/meilisearch-http/src/routes/key.rs +++ /dev/null @@ -1,23 +0,0 @@ -use actix_web::{web, HttpResponse}; -use serde::Serialize; - -use crate::extractors::authentication::{policies::*, GuardedData}; -use crate::Data; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("/keys").route(web::get().to(list))); -} - -#[derive(Serialize)] -struct KeysResponse { - private: Option, - public: Option, -} - -async fn list(data: GuardedData) -> HttpResponse { - let api_keys = data.api_keys.clone(); - HttpResponse::Ok().json(&KeysResponse { - private: api_keys.private, - public: api_keys.public, - }) -} diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 520949cd8..981d27e02 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,21 +1,27 @@ use std::time::Duration; -use actix_web::HttpResponse; +use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; +use log::debug; use serde::{Deserialize, Serialize}; use crate::error::ResponseError; +use crate::extractors::authentication::{policies::*, GuardedData}; use crate::index::{Settings, Unchecked}; use crate::index_controller::{UpdateMeta, UpdateResult, UpdateStatus}; +use crate::Data; -pub mod document; -pub mod dump; -pub mod health; -pub mod index; -pub mod key; -pub mod search; -pub mod settings; -pub mod stats; +mod dump; +mod indexes; + +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service(web::resource("/health").route(web::get().to(get_health))) + .service(web::scope("/dumps").configure(dump::configure)) + .service(web::resource("/keys").route(web::get().to(list_keys))) + .service(web::resource("/stats").route(web::get().to(get_stats))) + .service(web::resource("/version").route(web::get().to(get_version))) + .service(web::scope("/indexes").configure(indexes::configure)); +} #[derive(Debug, Clone, Serialize, Deserialize)] #[allow(clippy::large_enum_variant)] @@ -43,7 +49,6 @@ pub enum UpdateType { impl From<&UpdateStatus> for UpdateType { fn from(other: &UpdateStatus) -> Self { use milli::update::IndexDocumentsMethod::*; - match other.meta() { UpdateMeta::DocumentsAddition { method, .. } => { let number = match other { @@ -223,3 +228,53 @@ impl IndexUpdateResponse { pub async fn running() -> HttpResponse { HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" })) } + +async fn get_stats(data: GuardedData) -> Result { + let response = data.get_all_stats().await?; + + debug!("returns: {:?}", response); + Ok(HttpResponse::Ok().json(response)) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct VersionResponse { + commit_sha: String, + commit_date: String, + pkg_version: String, +} + +async fn get_version(_data: GuardedData) -> HttpResponse { + let commit_sha = match option_env!("COMMIT_SHA") { + Some("") | None => env!("VERGEN_SHA"), + Some(commit_sha) => commit_sha, + }; + let commit_date = match option_env!("COMMIT_DATE") { + Some("") | None => env!("VERGEN_COMMIT_DATE"), + Some(commit_date) => commit_date, + }; + + HttpResponse::Ok().json(VersionResponse { + commit_sha: commit_sha.to_string(), + commit_date: commit_date.to_string(), + pkg_version: env!("CARGO_PKG_VERSION").to_string(), + }) +} + +#[derive(Serialize)] +struct KeysResponse { + private: Option, + public: Option, +} + +pub async fn list_keys(data: GuardedData) -> HttpResponse { + let api_keys = data.api_keys.clone(); + HttpResponse::Ok().json(&KeysResponse { + private: api_keys.private, + public: api_keys.public, + }) +} + +pub async fn get_health() -> Result { + Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) +} diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs deleted file mode 100644 index a0078d76a..000000000 --- a/meilisearch-http/src/routes/stats.rs +++ /dev/null @@ -1,56 +0,0 @@ -use actix_web::{web, HttpResponse}; -use log::debug; -use serde::Serialize; - -use crate::error::ResponseError; -use crate::extractors::authentication::{policies::*, GuardedData}; -use crate::routes::IndexParam; -use crate::Data; - -pub fn services(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("/indexes/{index_uid}/stats").route(web::get().to(get_index_stats))) - .service(web::resource("/stats").route(web::get().to(get_stats))) - .service(web::resource("/version").route(web::get().to(get_version))); -} - -async fn get_index_stats( - data: GuardedData, - path: web::Path, -) -> Result { - let response = data.get_index_stats(path.index_uid.clone()).await?; - - debug!("returns: {:?}", response); - Ok(HttpResponse::Ok().json(response)) -} - -async fn get_stats(data: GuardedData) -> Result { - let response = data.get_all_stats().await?; - - debug!("returns: {:?}", response); - Ok(HttpResponse::Ok().json(response)) -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct VersionResponse { - commit_sha: String, - commit_date: String, - pkg_version: String, -} - -async fn get_version(_data: GuardedData) -> HttpResponse { - let commit_sha = match option_env!("COMMIT_SHA") { - Some("") | None => env!("VERGEN_SHA"), - Some(commit_sha) => commit_sha, - }; - let commit_date = match option_env!("COMMIT_DATE") { - Some("") | None => env!("VERGEN_COMMIT_DATE"), - Some(commit_date) => commit_date, - }; - - HttpResponse::Ok().json(VersionResponse { - commit_sha: commit_sha.to_string(), - commit_date: commit_date.to_string(), - pkg_version: env!("CARGO_PKG_VERSION").to_string(), - }) -} From 9681ffca524defa8889135c0e3857cfbe27b8f96 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 5 Jul 2021 10:10:17 +0200 Subject: [PATCH 12/53] change index create http code --- meilisearch-http/src/routes/indexes/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/indexes/mod.rs b/meilisearch-http/src/routes/indexes/mod.rs index 89fa366cb..260d97ecd 100644 --- a/meilisearch-http/src/routes/indexes/mod.rs +++ b/meilisearch-http/src/routes/indexes/mod.rs @@ -54,7 +54,7 @@ async fn create_index( ) -> Result { let body = body.into_inner(); let meta = data.create_index(body.uid, body.primary_key).await?; - Ok(HttpResponse::Ok().json(meta)) + Ok(HttpResponse::Created().json(meta)) } #[derive(Debug, Deserialize)] @@ -73,6 +73,7 @@ pub struct UpdateIndexResponse { updated_at: DateTime, primary_key: Option, } + async fn get_index( data: GuardedData, path: web::Path, From 2f3a439566a3257c6722d6b3eca367e84e559f1e Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 5 Jul 2021 11:21:02 +0200 Subject: [PATCH 13/53] fix tests --- meilisearch-http/tests/documents/get_documents.rs | 2 +- meilisearch-http/tests/index/create_index.rs | 6 +++--- meilisearch-http/tests/index/delete_index.rs | 2 +- meilisearch-http/tests/index/get_index.rs | 2 +- meilisearch-http/tests/index/stats.rs | 2 +- meilisearch-http/tests/index/update_index.rs | 6 +++--- meilisearch-http/tests/settings/get_settings.rs | 2 +- meilisearch-http/tests/stats/mod.rs | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index 945bd6b5c..14344db35 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -61,7 +61,7 @@ async fn get_no_documents() { let server = Server::new().await; let index = server.index("test"); let (_, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = index .get_all_documents(GetAllDocumentsOptions::default()) diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index e65908cb2..3c081f0ed 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -7,7 +7,7 @@ async fn create_index_no_primary_key() { let index = server.index("test"); let (response, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); assert_eq!(response["uid"], "test"); assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); @@ -23,7 +23,7 @@ async fn create_index_with_primary_key() { let index = server.index("test"); let (response, code) = index.create(Some("primary")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); assert_eq!(response["uid"], "test"); assert_eq!(response["name"], "test"); assert!(response.get("createdAt").is_some()); @@ -41,7 +41,7 @@ async fn create_existing_index() { let index = server.index("test"); let (_, code) = index.create(Some("primary")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (_response, code) = index.create(Some("primary")).await; assert_eq!(code, 400); diff --git a/meilisearch-http/tests/index/delete_index.rs b/meilisearch-http/tests/index/delete_index.rs index 6ce9587fc..10a6c0282 100644 --- a/meilisearch-http/tests/index/delete_index.rs +++ b/meilisearch-http/tests/index/delete_index.rs @@ -8,7 +8,7 @@ async fn create_and_delete_index() { let index = server.index("test"); let (_response, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (_response, code) = index.delete().await; diff --git a/meilisearch-http/tests/index/get_index.rs b/meilisearch-http/tests/index/get_index.rs index a6b22509e..ba26a8e3a 100644 --- a/meilisearch-http/tests/index/get_index.rs +++ b/meilisearch-http/tests/index/get_index.rs @@ -7,7 +7,7 @@ async fn create_and_get_index() { let index = server.index("test"); let (_, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = index.get().await; diff --git a/meilisearch-http/tests/index/stats.rs b/meilisearch-http/tests/index/stats.rs index 8494bbae3..3599e1605 100644 --- a/meilisearch-http/tests/index/stats.rs +++ b/meilisearch-http/tests/index/stats.rs @@ -8,7 +8,7 @@ async fn stats() { let index = server.index("test"); let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = index.stats().await; diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index c7d910b59..454b55fce 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -7,7 +7,7 @@ async fn update_primary_key() { let index = server.index("test"); let (_, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = index.update(Some("primary")).await; @@ -31,7 +31,7 @@ async fn update_nothing() { let index = server.index("test"); let (response, code) = index.create(None).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (update, code) = index.update(None).await; @@ -47,7 +47,7 @@ async fn update_existing_primary_key() { let index = server.index("test"); let (_response, code) = index.create(Some("primary")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (_update, code) = index.update(Some("primary2")).await; diff --git a/meilisearch-http/tests/settings/get_settings.rs b/meilisearch-http/tests/settings/get_settings.rs index 0b523eef3..837c90cc1 100644 --- a/meilisearch-http/tests/settings/get_settings.rs +++ b/meilisearch-http/tests/settings/get_settings.rs @@ -206,7 +206,7 @@ macro_rules! test_setting_routes { let server = Server::new().await; let index = server.index("test"); let (response, code) = index.create(None).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 201, "{}", response); let url = format!("/indexes/test/settings/{}", stringify!($setting) .chars() diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index aba860256..39c59fb27 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -28,7 +28,7 @@ async fn stats() { let index = server.index("test"); let (_, code) = index.create(Some("id")).await; - assert_eq!(code, 200); + assert_eq!(code, 201); let (response, code) = server.stats().await; From 066085f6f521e47b62dddd3ee39ac6241eda9e1f Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 5 Jul 2021 18:42:13 +0200 Subject: [PATCH 14/53] fix index deletion bug --- .../src/index_controller/update_actor/store/mod.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 99593cac8..8e85429ca 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -442,13 +442,16 @@ impl UpdateStore { while let Some(Ok(((_, uuid, _), pending))) = pendings.next() { if uuid == index_uuid { - unsafe { - pendings.del_current()?; - } let mut pending = pending.decode()?; if let Some(update_uuid) = pending.content.take() { uuids_to_remove.push(update_uuid); } + + // Invariant check: we can only delete the current entry when we don't hold + // references to it anymore. This must be done after we have retrieved its content. + unsafe { + pendings.del_current()?; + } } } From 520d37983c74f22a88af53420011d60d4948137e Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 6 Jul 2021 11:54:09 +0200 Subject: [PATCH 15/53] implement index search methods --- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/tests/common/index.rs | 29 +++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index aa0f2a974..5931d8d06 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -97,7 +97,7 @@ actix-rt = "2.1.0" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } mockall = "0.9.1" paste = "1.0.5" -serde_url_params = "0.2.0" +serde_url_params = "0.2.1" tempdir = "0.3.7" urlencoding = "1.1.1" diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 7d98d0733..021b11ec4 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{panic::{UnwindSafe, catch_unwind, resume_unwind}, time::Duration}; use actix_web::http::StatusCode; use paste::paste; @@ -185,6 +185,33 @@ impl Index<'_> { self.service.get(url).await } + pub async fn search(&self, query: Value, test: impl Fn(Value, StatusCode) + UnwindSafe + Clone) { + let (response, code) = self.search_post(query.clone()).await; + let t = test.clone(); + if let Err(e) = catch_unwind(move || t(response, code)) { + eprintln!("Error with post search"); + resume_unwind(e); + + } + + let (response, code) = self.search_get(query).await; + if let Err(e) = catch_unwind(move || test(response, code)) { + eprintln!("Error with get search"); + resume_unwind(e); + } + } + + pub async fn search_post(&self, query: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/search", self.uid); + self.service.post(url, query).await + } + + pub async fn search_get(&self, query: Value) -> (Value, StatusCode) { + let params = serde_url_params::to_string(&query).unwrap(); + let url = format!("/indexes/{}/search?{}", self.uid, params); + self.service.get(url).await + } + make_settings_test_routes!(distinct_attribute); } From 41e271974a905be99308bb1359e4541cf41f91f0 Mon Sep 17 00:00:00 2001 From: mpostma Date: Tue, 6 Jul 2021 11:54:37 +0200 Subject: [PATCH 16/53] add tests --- meilisearch-http/src/routes/indexes/search.rs | 10 + meilisearch-http/tests/common/index.rs | 13 +- meilisearch-http/tests/search/errors.rs | 28 +++ meilisearch-http/tests/search/mod.rs | 193 ++++++++++++++++++ 4 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 meilisearch-http/tests/search/errors.rs diff --git a/meilisearch-http/src/routes/indexes/search.rs b/meilisearch-http/src/routes/indexes/search.rs index b6820018e..d266c6f7a 100644 --- a/meilisearch-http/src/routes/indexes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -85,6 +85,11 @@ async fn search_with_url_query( debug!("called with params: {:?}", params); let query = params.into_inner().into(); let search_result = data.search(path.into_inner().index_uid, query).await?; + + // Tests that the nb_hits is always set to false + #[cfg(test)] + assert!(!search_result.exhaustive_nb_hits); + debug!("returns: {:?}", search_result); Ok(HttpResponse::Ok().json(search_result)) } @@ -98,6 +103,11 @@ async fn search_with_post( let search_result = data .search(path.into_inner().index_uid, params.into_inner()) .await?; + + // Tests that the nb_hits is always set to false + #[cfg(test)] + assert!(!search_result.exhaustive_nb_hits); + debug!("returns: {:?}", search_result); Ok(HttpResponse::Ok().json(search_result)) } diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 021b11ec4..cb790ba70 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -1,4 +1,7 @@ -use std::{panic::{UnwindSafe, catch_unwind, resume_unwind}, time::Duration}; +use std::{ + panic::{catch_unwind, resume_unwind, UnwindSafe}, + time::Duration, +}; use actix_web::http::StatusCode; use paste::paste; @@ -185,13 +188,17 @@ impl Index<'_> { self.service.get(url).await } - pub async fn search(&self, query: Value, test: impl Fn(Value, StatusCode) + UnwindSafe + Clone) { + /// Performs both GET and POST search queries + pub async fn search( + &self, + query: Value, + test: impl Fn(Value, StatusCode) + UnwindSafe + Clone, + ) { let (response, code) = self.search_post(query.clone()).await; let t = test.clone(); if let Err(e) = catch_unwind(move || t(response, code)) { eprintln!("Error with post search"); resume_unwind(e); - } let (response, code) = self.search_get(query).await; diff --git a/meilisearch-http/tests/search/errors.rs b/meilisearch-http/tests/search/errors.rs new file mode 100644 index 000000000..a09e8f76e --- /dev/null +++ b/meilisearch-http/tests/search/errors.rs @@ -0,0 +1,28 @@ +use crate::common::Server; +use serde_json::json; + +#[actix_rt::test] +async fn search_unexisting_index() { + let server = Server::new().await; + let index = server.index("test"); + + index + .search(json!({"q": "hello"}), |response, code| { + assert_eq!(code, 404, "{}", response); + assert_eq!(response["errorCode"], "index_not_found"); + }) + .await; +} + +#[actix_rt::test] +async fn search_unexisting_parameter() { + let server = Server::new().await; + let index = server.index("test"); + + index + .search(json!({"marin": "hello"}), |response, code| { + assert_eq!(code, 400, "{}", response); + assert_eq!(response["errorCode"], "bad_request"); + }) + .await; +} diff --git a/meilisearch-http/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs index 56ec6439c..b2a8f760c 100644 --- a/meilisearch-http/tests/search/mod.rs +++ b/meilisearch-http/tests/search/mod.rs @@ -1,2 +1,195 @@ // This modules contains all the test concerning search. Each particular feture of the search // should be tested in its own module to isolate tests and keep the tests readable. + +mod errors; + +use crate::common::Server; +use once_cell::sync::Lazy; +use serde_json::{json, Value}; + +static DOCUMENTS: Lazy = Lazy::new(|| { + json!([ + { + "title": "Shazam!", + "id": "287947" + }, + { + "title": "Captain Marvel", + "id": "299537" + }, + { + "title": "Escape Room", + "id": "522681" + }, + { "title": "How to Train Your Dragon: The Hidden World", "id": "166428" + }, + { + "title": "Glass", + "id": "450465" + } + ]) +}); + +#[actix_rt::test] +async fn simple_placeholder_search() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(0).await; + + index + .search(json!({}), |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 5); + }) + .await; +} + +#[actix_rt::test] +async fn simple_search() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(0).await; + + index + .search(json!({"q": "glass"}), |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1); + }) + .await; +} + +#[actix_rt::test] +async fn search_multiple_params() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(0).await; + + index + .search( + json!({ + "q": "glass", + "attributesToCrop": ["title:2"], + "attributesToHighlight": ["title"], + "limit": 1, + "offset": 0, + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1); + }, + ) + .await; +} + +#[actix_rt::test] +async fn search_with_filter_string_notation() { + let server = Server::new().await; + let index = server.index("test"); + + index + .update_settings(json!({"filterableAttributes": ["title"]})) + .await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(1).await; + + index + .search( + json!({ + "filter": "title = Glass" + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1); + }, + ) + .await; +} + +#[actix_rt::test] +async fn search_with_filter_array_notation() { + let server = Server::new().await; + let index = server.index("test"); + + index + .update_settings(json!({"filterableAttributes": ["title"]})) + .await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(1).await; + + let (response, code) = index + .search_post(json!({ + "filter": ["title = Glass"] + })) + .await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 1); + + let (response, code) = index + .search_post(json!({ + "filter": [["title = Glass", "title = \"Shazam!\"", "title = \"Escape Room\""]] + })) + .await; + assert_eq!(code, 200, "{}", response); + assert_eq!(response["hits"].as_array().unwrap().len(), 3); +} + +#[actix_rt::test] +async fn search_facet_distribution() { + let server = Server::new().await; + let index = server.index("test"); + + index + .update_settings(json!({"filterableAttributes": ["title"]})) + .await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(1).await; + + index + .search( + json!({ + "facetsDistribution": ["title"] + }), + |response, code| { + assert_eq!(code, 200, "{}", response); + let dist = response["facetsDistribution"].as_object().unwrap(); + assert_eq!(dist.len(), 1); + assert!(dist.get("title").is_some()); + }, + ) + .await; +} + +#[actix_rt::test] +async fn displayed_attributes() { + let server = Server::new().await; + let index = server.index("test"); + + index + .update_settings(json!({ "displayedAttributes": ["title"] })) + .await; + + let documents = DOCUMENTS.clone(); + index.add_documents(documents, None).await; + index.wait_update_id(1).await; + + let (response, code) = index + .search_post(json!({ "attributesToRetrieve": ["title", "id"] })) + .await; + assert_eq!(code, 200, "{}", response); + assert!(response["hits"].get("title").is_none()); +} From 587b837a6c9a6e0c5e339c13d6686f602f18e925 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Tue, 6 Jul 2021 21:41:26 +0200 Subject: [PATCH 17/53] Add MeiliSearch version to issue template --- .github/ISSUE_TEMPLATE/bug_report.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index dd84ea782..76c5536f8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -23,6 +23,9 @@ A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. +**MeiliSearch version:** +[e.g. v0.20.0] + **Desktop (please complete the following information):** - OS: [e.g. iOS] - Browser [e.g. chrome, safari] @@ -33,6 +36,6 @@ If applicable, add screenshots to help explain your problem. - OS: [e.g. iOS8.1] - Browser [e.g. stock browser, safari] - Version [e.g. 22] - + **Additional context** Add any other context about the problem here. From 83725a13305f03d27d07ed67b374da7cb5ec7976 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 7 Jul 2021 10:39:04 +0200 Subject: [PATCH 18/53] fix search permissions --- meilisearch-http/src/routes/indexes/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/indexes/search.rs b/meilisearch-http/src/routes/indexes/search.rs index d266c6f7a..c296b2f6a 100644 --- a/meilisearch-http/src/routes/indexes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -78,7 +78,7 @@ impl From for SearchQuery { } async fn search_with_url_query( - data: GuardedData, + data: GuardedData, path: web::Path, params: web::Query, ) -> Result { From b2ca600e7955c6be625eedbb9a4a43c4b6b0f1b9 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Wed, 7 Jul 2021 11:15:08 +0200 Subject: [PATCH 19/53] Remove unecessary questions --- .github/ISSUE_TEMPLATE/bug_report.md | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 76c5536f8..cea573edf 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -23,19 +23,8 @@ A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. -**MeiliSearch version:** -[e.g. v0.20.0] +**MeiliSearch version:** [e.g. v0.20.0] -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - **Additional context** -Add any other context about the problem here. +Additional information that may be relevant to the issue. +[e.g. architecture, device, OS, browser] \ No newline at end of file From a71fa25ebe9e1445f599c22472616eb6456690e0 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 7 Jul 2021 16:20:22 +0200 Subject: [PATCH 20/53] auth tests --- meilisearch-http/src/routes/dump.rs | 2 +- .../src/routes/indexes/documents.rs | 20 ++-- meilisearch-http/src/routes/indexes/mod.rs | 24 ++--- meilisearch-http/src/routes/indexes/search.rs | 4 +- .../src/routes/indexes/settings.rs | 14 +-- .../src/routes/indexes/updates.rs | 6 +- meilisearch-http/src/routes/mod.rs | 94 +++++++++++++++++++ 7 files changed, 129 insertions(+), 35 deletions(-) diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index 18b143fa3..72bc55986 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -11,7 +11,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status))); } -async fn create_dump(data: GuardedData) -> Result { +pub async fn create_dump(data: GuardedData) -> Result { let res = data.create_dump().await?; debug!("returns: {:?}", res); diff --git a/meilisearch-http/src/routes/indexes/documents.rs b/meilisearch-http/src/routes/indexes/documents.rs index a62a85f1b..a4bf465b5 100644 --- a/meilisearch-http/src/routes/indexes/documents.rs +++ b/meilisearch-http/src/routes/indexes/documents.rs @@ -49,7 +49,7 @@ fn guard_json(head: &actix_web::dev::RequestHead) -> bool { } #[derive(Deserialize)] -struct DocumentParam { +pub struct DocumentParam { index_uid: String, document_id: String, } @@ -71,7 +71,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } -async fn get_document( +pub async fn get_document( data: GuardedData, path: web::Path, ) -> Result { @@ -84,7 +84,7 @@ async fn get_document( Ok(HttpResponse::Ok().json(document)) } -async fn delete_document( +pub async fn delete_document( data: GuardedData, path: web::Path, ) -> Result { @@ -97,13 +97,13 @@ async fn delete_document( #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -struct BrowseQuery { +pub struct BrowseQuery { offset: Option, limit: Option, attributes_to_retrieve: Option, } -async fn get_all_documents( +pub async fn get_all_documents( data: GuardedData, path: web::Path, params: web::Query, @@ -134,13 +134,13 @@ async fn get_all_documents( #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -struct UpdateDocumentsQuery { +pub struct UpdateDocumentsQuery { primary_key: Option, } /// Route used when the payload type is "application/json" /// Used to add or replace documents -async fn add_documents( +pub async fn add_documents( data: GuardedData, path: web::Path, params: web::Query, @@ -163,7 +163,7 @@ async fn add_documents( /// Route used when the payload type is "application/json" /// Used to add or replace documents -async fn update_documents( +pub async fn update_documents( data: GuardedData, path: web::Path, params: web::Query, @@ -184,7 +184,7 @@ async fn update_documents( Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() }))) } -async fn delete_documents( +pub async fn delete_documents( data: GuardedData, path: web::Path, body: web::Json>, @@ -204,7 +204,7 @@ async fn delete_documents( Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } -async fn clear_all_documents( +pub async fn clear_all_documents( data: GuardedData, path: web::Path, ) -> Result { diff --git a/meilisearch-http/src/routes/indexes/mod.rs b/meilisearch-http/src/routes/indexes/mod.rs index 260d97ecd..8314bf032 100644 --- a/meilisearch-http/src/routes/indexes/mod.rs +++ b/meilisearch-http/src/routes/indexes/mod.rs @@ -8,10 +8,10 @@ use crate::extractors::authentication::{policies::*, GuardedData}; use crate::routes::IndexParam; use crate::Data; -mod documents; -mod search; -mod settings; -mod updates; +pub mod documents; +pub mod search; +pub mod settings; +pub mod updates; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( @@ -35,7 +35,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { ); } -async fn list_indexes(data: GuardedData) -> Result { +pub async fn list_indexes(data: GuardedData) -> Result { let indexes = data.list_indexes().await?; debug!("returns: {:?}", indexes); Ok(HttpResponse::Ok().json(indexes)) @@ -43,12 +43,12 @@ async fn list_indexes(data: GuardedData) -> Result, } -async fn create_index( +pub async fn create_index( data: GuardedData, body: web::Json, ) -> Result { @@ -59,7 +59,7 @@ async fn create_index( #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -struct UpdateIndexRequest { +pub struct UpdateIndexRequest { uid: Option, primary_key: Option, } @@ -74,7 +74,7 @@ pub struct UpdateIndexResponse { primary_key: Option, } -async fn get_index( +pub async fn get_index( data: GuardedData, path: web::Path, ) -> Result { @@ -83,7 +83,7 @@ async fn get_index( Ok(HttpResponse::Ok().json(meta)) } -async fn update_index( +pub async fn update_index( data: GuardedData, path: web::Path, body: web::Json, @@ -97,7 +97,7 @@ async fn update_index( Ok(HttpResponse::Ok().json(meta)) } -async fn delete_index( +pub async fn delete_index( data: GuardedData, path: web::Path, ) -> Result { @@ -105,7 +105,7 @@ async fn delete_index( Ok(HttpResponse::NoContent().finish()) } -async fn get_index_stats( +pub async fn get_index_stats( data: GuardedData, path: web::Path, ) -> Result { diff --git a/meilisearch-http/src/routes/indexes/search.rs b/meilisearch-http/src/routes/indexes/search.rs index c296b2f6a..8a2f4ae1d 100644 --- a/meilisearch-http/src/routes/indexes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -77,7 +77,7 @@ impl From for SearchQuery { } } -async fn search_with_url_query( +pub async fn search_with_url_query( data: GuardedData, path: web::Path, params: web::Query, @@ -94,7 +94,7 @@ async fn search_with_url_query( Ok(HttpResponse::Ok().json(search_result)) } -async fn search_with_post( +pub async fn search_with_post( data: GuardedData, path: web::Path, params: web::Json, diff --git a/meilisearch-http/src/routes/indexes/settings.rs b/meilisearch-http/src/routes/indexes/settings.rs index 272060bf5..47f18a68a 100644 --- a/meilisearch-http/src/routes/indexes/settings.rs +++ b/meilisearch-http/src/routes/indexes/settings.rs @@ -9,7 +9,7 @@ use crate::{error::ResponseError, index::Unchecked}; #[macro_export] macro_rules! make_setting_route { ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => { - mod $attr { + pub mod $attr { use log::debug; use actix_web::{web, HttpResponse, Resource}; @@ -18,7 +18,7 @@ macro_rules! make_setting_route { use crate::index::Settings; use crate::extractors::authentication::{GuardedData, policies::*}; - async fn delete( + pub async fn delete( data: GuardedData, index_uid: web::Path, ) -> Result { @@ -32,7 +32,7 @@ macro_rules! make_setting_route { Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } - async fn update( + pub async fn update( data: GuardedData, index_uid: actix_web::web::Path, body: actix_web::web::Json>, @@ -47,7 +47,7 @@ macro_rules! make_setting_route { Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) } - async fn get( + pub async fn get( data: GuardedData, index_uid: actix_web::web::Path, ) -> std::result::Result { @@ -135,7 +135,7 @@ generate_configure!( ranking_rules ); -async fn update_all( +pub async fn update_all( data: GuardedData, index_uid: web::Path, body: web::Json>, @@ -149,7 +149,7 @@ async fn update_all( Ok(HttpResponse::Accepted().json(json)) } -async fn get_all( +pub async fn get_all( data: GuardedData, index_uid: web::Path, ) -> Result { @@ -158,7 +158,7 @@ async fn get_all( Ok(HttpResponse::Ok().json(settings)) } -async fn delete_all( +pub async fn delete_all( data: GuardedData, index_uid: web::Path, ) -> Result { diff --git a/meilisearch-http/src/routes/indexes/updates.rs b/meilisearch-http/src/routes/indexes/updates.rs index 735736e97..471636abf 100644 --- a/meilisearch-http/src/routes/indexes/updates.rs +++ b/meilisearch-http/src/routes/indexes/updates.rs @@ -31,12 +31,12 @@ pub struct UpdateIndexResponse { } #[derive(Deserialize)] -struct UpdateParam { +pub struct UpdateParam { index_uid: String, update_id: u64, } -async fn get_update_status( +pub async fn get_update_status( data: GuardedData, path: web::Path, ) -> Result { @@ -49,7 +49,7 @@ async fn get_update_status( Ok(HttpResponse::Ok().json(meta)) } -async fn get_all_updates_status( +pub async fn get_all_updates_status( data: GuardedData, path: web::Path, ) -> Result { diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 981d27e02..fd30cd519 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -278,3 +278,97 @@ pub async fn list_keys(data: GuardedData) -> HttpResponse { pub async fn get_health() -> Result { Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) } + +#[cfg(test)] +mod test { + use super::*; + use crate::data::Data; + use crate::extractors::authentication::GuardedData; + + /// A type implemented for a route that uses a authentication policy `Policy`. + /// + /// This trait is used for regression testing of route authenticaton policies. + trait Is {} + + macro_rules! impl_is_policy { + ($($param:ident)*) => { + impl Is for Func + where Func: Fn(GuardedData, $($param,)*) -> Res {} + + }; + } + + impl_is_policy! {} + impl_is_policy! {A} + impl_is_policy! {A B} + impl_is_policy! {A B C} + impl_is_policy! {A B C D} + + /// Emits a compile error if a route doesn't have the correct authentication policy. + /// + /// This works by trying to cast the route function into a Is type, where Policy it + /// the authentication policy defined for the route. + macro_rules! test_auth_routes { + ($($policy:ident => { $($route:expr,)*})*) => { + #[test] + fn test_auth() { + $($(let _: &dyn Is<$policy, _> = &$route;)*)* + } + }; + } + + test_auth_routes! { + Public => { + indexes::search::search_with_url_query, + indexes::search::search_with_post, + + indexes::documents::get_document, + indexes::documents::get_all_documents, + } + Private => { + get_stats, + get_version, + + indexes::create_index, + indexes::list_indexes, + indexes::get_index_stats, + indexes::delete_index, + indexes::update_index, + indexes::get_index, + + dump::create_dump, + + indexes::settings::filterable_attributes::get, + indexes::settings::displayed_attributes::get, + indexes::settings::searchable_attributes::get, + indexes::settings::stop_words::get, + indexes::settings::synonyms::get, + indexes::settings::distinct_attribute::get, + indexes::settings::filterable_attributes::update, + indexes::settings::displayed_attributes::update, + indexes::settings::searchable_attributes::update, + indexes::settings::stop_words::update, + indexes::settings::synonyms::update, + indexes::settings::distinct_attribute::update, + indexes::settings::filterable_attributes::delete, + indexes::settings::displayed_attributes::delete, + indexes::settings::searchable_attributes::delete, + indexes::settings::stop_words::delete, + indexes::settings::synonyms::delete, + indexes::settings::distinct_attribute::delete, + indexes::settings::delete_all, + indexes::settings::get_all, + indexes::settings::update_all, + + indexes::documents::clear_all_documents, + indexes::documents::delete_documents, + indexes::documents::update_documents, + indexes::documents::add_documents, + indexes::documents::delete_document, + + indexes::updates::get_all_updates_status, + indexes::updates::get_update_status, + } + Admin => { list_keys, } + } +} From 076d8fbb843710ac9d571b42bd901f3b86031592 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 7 Jul 2021 17:07:20 +0200 Subject: [PATCH 21/53] add cache to the ci --- .github/workflows/rust.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6e3d939e2..1b361c9bc 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -22,6 +22,13 @@ jobs: os: [ubuntu-18.04, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo + ./target + key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: @@ -38,6 +45,13 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo + ./target + key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -55,6 +69,13 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo + ./target + key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} - uses: actions-rs/toolchain@v1 with: profile: minimal From f49a01703a87db240967ccb633fc1bcfa8bae8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Fri, 9 Jul 2021 16:11:37 +0200 Subject: [PATCH 22/53] Show the filterable and not the faceted attributes in the settings --- meilisearch-http/src/index/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 7227f8d35..85eff2746 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -69,7 +69,7 @@ impl Index { .searchable_fields(&txn)? .map(|fields| fields.into_iter().map(String::from).collect()); - let faceted_attributes = self.faceted_fields(&txn)?.into_iter().collect(); + let filterable_attributes = self.filterable_fields(&txn)?.into_iter().collect(); let criteria = self .criteria(&txn)? @@ -102,7 +102,7 @@ impl Index { Ok(Settings { displayed_attributes: Some(displayed_attributes), searchable_attributes: Some(searchable_attributes), - filterable_attributes: Some(Some(faceted_attributes)), + filterable_attributes: Some(Some(filterable_attributes)), ranking_rules: Some(Some(criteria)), stop_words: Some(Some(stop_words)), distinct_attribute: Some(distinct_field), From 5d4a0ac8442d9fba4536c00627c2b46db127065a Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 21 Jul 2021 11:50:54 +0200 Subject: [PATCH 23/53] sentry was never sending anything --- meilisearch-http/src/main.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 5638c453f..dedb6cc26 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -49,12 +49,12 @@ async fn main() -> Result<(), MainError> { release: sentry::release_name!(), dsn: Some(SENTRY_DSN.parse()?), before_send: Some(Arc::new(|event| { - event - .message - .as_ref() - .map(|msg| msg.to_lowercase().contains("no space left on device")) - .unwrap_or(false) - .then(|| event) + if matches!(event.message, Some(ref msg) if msg.to_lowercase().contains("no space left on device")) + { + None + } else { + Some(event) + } })), ..Default::default() }); From f79b8287f5bd43117e64ab6c04537b1c9d16755c Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 26 Jul 2021 15:25:30 +0200 Subject: [PATCH 24/53] update vergen --- Cargo.lock | 109 ++++++++++++++++++++++++----- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/build.rs | 9 +-- meilisearch-http/src/main.rs | 6 +- meilisearch-http/src/routes/mod.rs | 4 +- 5 files changed, 98 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0cc33b2e9..edb937d85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -831,6 +831,26 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "enum-iterator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6" +dependencies = [ + "enum-iterator-derive", +] + +[[package]] +name = "enum-iterator-derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" +dependencies = [ + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", +] + [[package]] name = "env_logger" version = "0.8.4" @@ -1067,12 +1087,37 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "getset" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b328c01a4d71d2d8173daa93562a73ab0fe85616876f02500f53d82948c504" +dependencies = [ + "proc-macro-error", + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", +] + [[package]] name = "gimli" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" +[[package]] +name = "git2" +version = "0.13.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9831e983241f8c5591ed53f17d874833e2fa82cac2625f3888c50cbfe136cba" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url", +] + [[package]] name = "glob" version = "0.3.0" @@ -1437,6 +1482,30 @@ version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" +[[package]] +name = "libgit2-sys" +version = "0.12.21+1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86271bacd72b2b9e854c3dcfb82efd538f15f870e4c11af66900effb462f6825" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libz-sys" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -2460,15 +2529,6 @@ dependencies = [ "semver 0.11.0", ] -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver 1.0.3", -] - [[package]] name = "rustls" version = "0.19.1" @@ -2482,6 +2542,12 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustversion" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" + [[package]] name = "ryu" version = "1.0.5" @@ -2531,12 +2597,6 @@ dependencies = [ "semver-parser 0.10.2", ] -[[package]] -name = "semver" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f3aac57ee7f3272d8395c6e4f502f434f0e289fcd62876f70daa008c20dcabe" - [[package]] name = "semver-parser" version = "0.7.0" @@ -3299,6 +3359,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vec_map" version = "0.8.2" @@ -3307,13 +3373,18 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "vergen" -version = "3.2.0" +version = "5.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a" +checksum = "542f37b4798c879409865dde7908e746d836f77839c3a6bea5c8b4e4dcf6620b" dependencies = [ - "bitflags", + "anyhow", + "cfg-if 1.0.0", "chrono", - "rustc_version 0.4.0", + "enum-iterator", + "getset", + "git2", + "rustversion", + "thiserror", ] [[package]] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 5931d8d06..3b1f5ff57 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -18,7 +18,7 @@ hex = { version = "0.4.3", optional = true } reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false, optional = true } sha-1 = { version = "0.9.4", optional = true } tempfile = { version = "3.1.0", optional = true } -vergen = "3.1.0" +vergen = { version = "5.1.13", default-features = false, features = ["build", "git"] } zip = { version = "0.5.12", optional = true } [dependencies] diff --git a/meilisearch-http/build.rs b/meilisearch-http/build.rs index 557e04fe7..e33acc892 100644 --- a/meilisearch-http/build.rs +++ b/meilisearch-http/build.rs @@ -1,12 +1,7 @@ -use vergen::{generate_cargo_keys, ConstantsFlags}; +use vergen::{vergen, Config}; fn main() { - // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag - let mut flags = ConstantsFlags::all(); - flags.toggle(ConstantsFlags::SEMVER_FROM_CARGO_PKG); - - // Generate the 'cargo:' key output - generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!"); + vergen(Config::default()).unwrap(); #[cfg(feature = "mini-dashboard")] mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets"); diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index dedb6cc26..033c79408 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -105,11 +105,11 @@ async fn run_http(data: Data, opt: Opt) -> Result<(), Box pub fn print_launch_resume(opt: &Opt, data: &Data) { let commit_sha = match option_env!("COMMIT_SHA") { - Some("") | None => env!("VERGEN_SHA"), + Some("") | None => env!("VERGEN_GIT_SHA"), Some(commit_sha) => commit_sha, }; - let commit_date = match option_env!("COMMIT_DATE") { - Some("") | None => env!("VERGEN_COMMIT_DATE"), + let commit_date = match dbg!(option_env!("COMMIT_DATE")) { + Some("") | None => dbg!(env!("VERGEN_GIT_COMMIT_TIMESTAMP")), Some(commit_date) => commit_date, }; diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index fd30cd519..184616892 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -246,11 +246,11 @@ struct VersionResponse { async fn get_version(_data: GuardedData) -> HttpResponse { let commit_sha = match option_env!("COMMIT_SHA") { - Some("") | None => env!("VERGEN_SHA"), + Some("") | None => env!("VERGEN_GIT_SHA"), Some(commit_sha) => commit_sha, }; let commit_date = match option_env!("COMMIT_DATE") { - Some("") | None => env!("VERGEN_COMMIT_DATE"), + Some("") | None => env!("VERGEN_GIT_COMMIT_TIMESTAMP"), Some(commit_date) => commit_date, }; From d3cd7e92d1a3ebfee83ca3eb1e14b4d358c2dc7c Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Tue, 27 Jul 2021 11:44:20 +0200 Subject: [PATCH 25/53] Update mini-dashboard version to v0.1.4 --- meilisearch-http/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 3b1f5ff57..f0186801a 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -119,5 +119,5 @@ default = ["analytics", "mini-dashboard"] jemallocator = "0.3.2" [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.3/build.zip" -sha1 = "fea1780e13d8e570e35a1921e7a45cabcd501d5e" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.4/build.zip" +sha1 = "750e8a8e56cfa61fbf9ead14b08a5f17ad3f3d37" From 1a1046a0ef7d2a19017c0e826b55c81fcdcc9912 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 27 Jul 2021 16:01:06 +0200 Subject: [PATCH 26/53] start writing some documentation for newcomers --- .../src/index_controller/dump_actor/mod.rs | 2 +- meilisearch-http/src/lib.rs | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index a73740b02..c8aad6815 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -37,7 +37,7 @@ pub trait DumpActorHandle { async fn create_dump(&self) -> Result; /// Return the status of an already created dump - /// Implementation: [handle_impl::DumpActorHandleImpl::dump_status] + /// Implementation: [handle_impl::DumpActorHandleImpl::dump_info] async fn dump_info(&self, uid: String) -> Result; } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index bbc7cefab..89a578087 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -1,3 +1,43 @@ +//! # MeiliSearch +//! Hello there, future contributors. If you are here and see this code, it's probably because you want to add a super new fancy feature in MeiliSearch or fix a bug and first of all, thank you for that! +//! +//! To help you in this task, we'll try to do a little overview of the project. +//! ## Milli +//! [Milli](https://github.com/meilisearch/milli) is the core library of MeiliSearch, it's where we actually index documents and perform searches. It's aim is to do theses two tasks as fast as possible. You can give an update to milli and it'll uses as many cores as provided to perform it as fast as possible. Nothing more. You can perform searches at the same time (search only use one core). +//! As you can see we're missing quite a lot of features here, milli does not handle multiples indexes, it can't queue updates, it doesn't provide any web / API frontend, it doesn't implements dumps or snapshots, etc... +//! +//! ## `Index` module +//! The [index] module is what encapsulate one milli index. It abstracts over its transaction and isolate a task that can be run into a thread. This is the unit of interaction with milli. +//! If you add a feature to milli, you'll probably need to add it in this module too before being able to expose it to the rest of meilisearch. +//! +//! ## `IndexController` module +//! To handle multiple indexes we created an [index_controller]. It's in charge of creating new indexes, keeping references to all its indexes forward asynchronous updates to its indexes and provide an API to synchronously search in its indexes. +//! To achieves this goal, we use an [actor model](https://en.wikipedia.org/wiki/Actor_model). +//! +//! ### The actor model +//! Every actor is composed of at least three files: +//! - `mod.rs` declare and import all the files used by the actor. We also describe the interface (= all the methods) used to interact with the actor. If you are not modifying anything inside of an actor, this is usually all you need to see. +//! - `handle_impl.rs` implements the interface described in the `mod.rs`; in reality, there is no code logic in this file. Every method is only wrapping its parameters in a structure that is sent to the actor. This is useful for test and futureproofing. +//! - `message.rs` contains an enum that describes all the interactions you can have with the actor. +//! - `actor.rs` is used to create and execute the actor. It's where we'll write the loop looking for new messages and actually perform the tasks. +//! +//! MeiliSearch currently uses four actors: +//! - [`uuid_resolver`](index_controller/uuid_resolver/index.html) hold the association between the user-provided indexes name and the internal [`uuid`](https://en.wikipedia.org/wiki/Universally_unique_identifier) representation we use. +//! - [`index_actor`](index_controller::index_actor) is our representation of multiples indexes. Any request made to MeiliSearch that needs to talk to milli will pass through this actor. +//! - [`update_actor`](index_controller/update_actor/index.html) is in charge of indexes updates. Since updates can take a long time to receive and process, we need to: +//! 1. Store them as fast as possible so we can continue to receive other updates even if nothing has been processed +//! 2. Feed the `index_actor` with a new update every time it finished its current job. +//! - [`dump_actor`](index_controller/dump_actor/index.html) this actor handle the [dumps](https://docs.meilisearch.com/reference/api/dump.html). It needs to contact all the others actors and create a dump of everything that was currently happening. +//! +//! ## Data module +//! The [data] module provide an unified interface to communicate with the index controller and other services (snapshot, dumps, ...), initialize the MeiliSearch instance +//! +//! ## Http server +//! To handle the web and API part, we are using [actix-web](https://docs.rs/actix-web/); you can find all routes in the [routes] module. +//! Currently the configuration of actix-web is made in the [lib.rs](crate). +//! Most of the routes uses [extractors] to handle the authentication. + +#![allow(rustdoc::private_intra_doc_links)] pub mod data; #[macro_use] pub mod error; From e5dabf265ac749a56269ce6c128e1541b5146fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 28 Jul 2021 10:52:47 +0200 Subject: [PATCH 27/53] Update milli version to v0.8.0 --- Cargo.lock | 57 +++++++++++++++++++++++----- meilisearch-http/Cargo.toml | 6 +-- meilisearch-http/src/index/mod.rs | 2 +- meilisearch-http/src/index/search.rs | 14 +++---- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edb937d85..b1e2ac465 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -620,6 +620,17 @@ dependencies = [ "vec_map", ] +[[package]] +name = "concat-arrays" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df715824eb382e34b7afb7463b0247bf41538aeba731fba05241ecdb5dc3747" +dependencies = [ + "proc-macro2 1.0.27", + "quote 1.0.9", + "syn 1.0.73", +] + [[package]] name = "const_fn" version = "0.4.8" @@ -1191,7 +1202,7 @@ dependencies = [ [[package]] name = "heed" version = "0.12.0" -source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551" dependencies = [ "byteorder", "heed-traits", @@ -1209,12 +1220,12 @@ dependencies = [ [[package]] name = "heed-traits" version = "0.7.0" -source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551" [[package]] name = "heed-types" version = "0.7.2" -source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551" dependencies = [ "bincode", "heed-traits", @@ -1647,7 +1658,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.3", "memmap", "milli", "mime", @@ -1699,7 +1710,23 @@ dependencies = [ "once_cell", "slice-group-by", "unicode-segmentation", - "whatlang", + "whatlang 0.9.0", +] + +[[package]] +name = "meilisearch-tokenizer" +version = "0.2.4" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.4#135d08dce465a756abaf6a1bcad70f315bda99b9" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang 0.12.0", ] [[package]] @@ -1729,12 +1756,13 @@ dependencies = [ [[package]] name = "milli" -version = "0.7.2" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.7.2#007fec21fcb3accad807aa2f2cf66c168afb043f" +version = "0.8.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.8.0#200e98c211510750e2ead475b8571b533ac6caa1" dependencies = [ "bstr", "byteorder", "chrono", + "concat-arrays", "csv", "either", "flate2", @@ -1748,7 +1776,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.4", "memmap", "obkv", "once_cell", @@ -1910,9 +1938,9 @@ dependencies = [ [[package]] name = "obkv" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8" +checksum = "f69e48cd7c8e5bb52a1da1287fdbfd877c32673176583ce664cd63b201aba385" [[package]] name = "once_cell" @@ -3532,6 +3560,15 @@ dependencies = [ "hashbrown 0.7.2", ] +[[package]] +name = "whatlang" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a346d2eb29c03618693ed24a29d1acd0c3f2cb08ae58b9669d7461e033cf703" +dependencies = [ + "hashbrown 0.7.2", +] + [[package]] name = "whoami" version = "1.1.2" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index f0186801a..6191ca2bb 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -42,7 +42,7 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" } +heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" } http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } itertools = "0.10.0" @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.2" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.8.0" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" @@ -73,7 +73,7 @@ thiserror = "1.0.24" tokio = { version = "1", features = ["full"] } uuid = { version = "0.8.2", features = ["serde"] } walkdir = "2.3.2" -obkv = "0.1.1" +obkv = "0.2.0" pin-project = "1.0.7" whoami = { version = "1.1.2", optional = true } reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true } diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 85eff2746..4afec5460 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -174,7 +174,7 @@ impl Index { txn: &heed::RoTxn, attributes_to_retrieve: &Option>, fields_ids_map: &milli::FieldsIdsMap, - ) -> Result> { + ) -> Result> { let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { Some(ids) => ids.into_iter().collect::>(), None => fields_ids_map.iter().map(|(id, _)| id).collect(), diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 537dffc0f..91eb2a2ca 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -281,9 +281,9 @@ fn compute_formatted_options( attr_to_highlight: &HashSet, attr_to_crop: &[String], query_crop_length: usize, - to_retrieve_ids: &BTreeSet, + to_retrieve_ids: &BTreeSet, fields_ids_map: &FieldsIdsMap, - displayed_ids: &BTreeSet, + displayed_ids: &BTreeSet, ) -> BTreeMap { let mut formatted_options = BTreeMap::new(); @@ -314,7 +314,7 @@ fn add_highlight_to_formatted_options( formatted_options: &mut BTreeMap, attr_to_highlight: &HashSet, fields_ids_map: &FieldsIdsMap, - displayed_ids: &BTreeSet, + displayed_ids: &BTreeSet, ) { for attr in attr_to_highlight { let new_format = FormatOptions { @@ -342,7 +342,7 @@ fn add_crop_to_formatted_options( attr_to_crop: &[String], crop_length: usize, fields_ids_map: &FieldsIdsMap, - displayed_ids: &BTreeSet, + displayed_ids: &BTreeSet, ) { for attr in attr_to_crop { let mut split = attr.rsplitn(2, ':'); @@ -382,7 +382,7 @@ fn add_crop_to_formatted_options( fn add_non_formatted_ids_to_formatted_options( formatted_options: &mut BTreeMap, - to_retrieve_ids: &BTreeSet, + to_retrieve_ids: &BTreeSet, ) { for id in to_retrieve_ids { formatted_options.entry(*id).or_insert(FormatOptions { @@ -395,7 +395,7 @@ fn add_non_formatted_ids_to_formatted_options( fn make_document( attributes_to_retrieve: &BTreeSet, field_ids_map: &FieldsIdsMap, - obkv: obkv::KvReader, + obkv: obkv::KvReaderU16, ) -> Result { let mut document = Document::new(); @@ -418,7 +418,7 @@ fn make_document( fn format_fields>( field_ids_map: &FieldsIdsMap, - obkv: obkv::KvReader, + obkv: obkv::KvReaderU16, formatter: &Formatter, matching_words: &impl Matcher, formatted_options: &BTreeMap, From 9bdd040dd043d51ba89562e86b71e9688f46b600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 28 Jul 2021 12:08:19 +0200 Subject: [PATCH 28/53] Update meilisearch-http/src/index/mod.rs Co-authored-by: marin --- meilisearch-http/src/index/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 4afec5460..e04d47a71 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -174,7 +174,7 @@ impl Index { txn: &heed::RoTxn, attributes_to_retrieve: &Option>, fields_ids_map: &milli::FieldsIdsMap, - ) -> Result> { + ) -> Result> { let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { Some(ids) => ids.into_iter().collect::>(), None => fields_ids_map.iter().map(|(id, _)| id).collect(), From 7e0eed5772b38a6199d8470d6bdbe178dd771b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 28 Jul 2021 12:08:24 +0200 Subject: [PATCH 29/53] Update meilisearch-http/src/index/search.rs Co-authored-by: marin --- meilisearch-http/src/index/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 91eb2a2ca..479bf6dc4 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -283,7 +283,7 @@ fn compute_formatted_options( query_crop_length: usize, to_retrieve_ids: &BTreeSet, fields_ids_map: &FieldsIdsMap, - displayed_ids: &BTreeSet, + displayed_ids: &BTreeSet, ) -> BTreeMap { let mut formatted_options = BTreeMap::new(); From feb069f6049494c2c386856b4d763a2bcd7dc420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 28 Jul 2021 12:08:28 +0200 Subject: [PATCH 30/53] Update meilisearch-http/src/index/search.rs Co-authored-by: marin --- meilisearch-http/src/index/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 479bf6dc4..c0269cc9a 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -342,7 +342,7 @@ fn add_crop_to_formatted_options( attr_to_crop: &[String], crop_length: usize, fields_ids_map: &FieldsIdsMap, - displayed_ids: &BTreeSet, + displayed_ids: &BTreeSet, ) { for attr in attr_to_crop { let mut split = attr.rsplitn(2, ':'); From f9d337b320e10e3bbbbc405d013b183b802a5689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 28 Jul 2021 12:08:36 +0200 Subject: [PATCH 31/53] Update meilisearch-http/src/index/search.rs Co-authored-by: marin --- meilisearch-http/src/index/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index c0269cc9a..1b3769890 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -382,7 +382,7 @@ fn add_crop_to_formatted_options( fn add_non_formatted_ids_to_formatted_options( formatted_options: &mut BTreeMap, - to_retrieve_ids: &BTreeSet, + to_retrieve_ids: &BTreeSet, ) { for id in to_retrieve_ids { formatted_options.entry(*id).or_insert(FormatOptions { From 6a6e2a8cd1eacadbe79810ab356cd5d6135ac6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 28 Jul 2021 12:08:51 +0200 Subject: [PATCH 32/53] Update meilisearch-http/src/index/search.rs Co-authored-by: marin --- meilisearch-http/src/index/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 1b3769890..afa917701 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -314,7 +314,7 @@ fn add_highlight_to_formatted_options( formatted_options: &mut BTreeMap, attr_to_highlight: &HashSet, fields_ids_map: &FieldsIdsMap, - displayed_ids: &BTreeSet, + displayed_ids: &BTreeSet, ) { for attr in attr_to_highlight { let new_format = FormatOptions { From 1df8f041bd59ee88b8a9ca91a732bf93fc01189d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 28 Jul 2021 12:10:25 +0200 Subject: [PATCH 33/53] Update meilisearch-http/src/index/search.rs Co-authored-by: marin --- meilisearch-http/src/index/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index afa917701..b2129ca53 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -281,7 +281,7 @@ fn compute_formatted_options( attr_to_highlight: &HashSet, attr_to_crop: &[String], query_crop_length: usize, - to_retrieve_ids: &BTreeSet, + to_retrieve_ids: &BTreeSet, fields_ids_map: &FieldsIdsMap, displayed_ids: &BTreeSet, ) -> BTreeMap { From 80916baa21afa7d7908d8db063e276ff1cdbde7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 28 Jul 2021 12:20:13 +0200 Subject: [PATCH 34/53] Add FieldId in import --- meilisearch-http/src/index/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index e04d47a71..4c963f1f5 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -6,7 +6,7 @@ use std::path::Path; use std::sync::Arc; use heed::{EnvOpenOptions, RoTxn}; -use milli::obkv_to_json; +use milli::{obkv_to_json, FieldId}; use serde::{de::Deserializer, Deserialize}; use serde_json::{Map, Value}; From c55f73bbc34d7067b2d02f9328f0b3e305e339d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 28 Jul 2021 17:29:40 +0200 Subject: [PATCH 35/53] Remove ARMv7 support --- .github/workflows/publish-binaries.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 601a8dbf4..19c3977fe 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -37,30 +37,6 @@ jobs: asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} - publish-armv7: - name: Publish for ARMv7 - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v1.0.0 - - uses: uraimo/run-on-arch-action@v1.0.7 - id: runcmd - with: - architecture: armv7 - distribution: ubuntu18.04 - run: | - apt update - apt install -y curl gcc make - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable - source $HOME/.cargo/env - cargo build --release --locked - - name: Upload the binary to release - uses: svenstaro/upload-release-action@v1-release - with: - repo_token: ${{ secrets.PUBLISH_TOKEN }} - file: target/release/meilisearch - asset_name: meilisearch-linux-armv7 - tag: ${{ github.ref }} - publish-armv8: name: Publish for ARMv8 runs-on: ubuntu-18.04 From 243233f652743d4df526f0b00a78c9e5824896a8 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 28 Jul 2021 19:12:40 +0200 Subject: [PATCH 36/53] import .git to docker to fix vergen --- .dockerignore | 1 - meilisearch-http/src/main.rs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 364510117..8c6bdbdeb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ target Dockerfile .dockerignore -.git .gitignore diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 033c79408..33c99191e 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -108,8 +108,8 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { Some("") | None => env!("VERGEN_GIT_SHA"), Some(commit_sha) => commit_sha, }; - let commit_date = match dbg!(option_env!("COMMIT_DATE")) { - Some("") | None => dbg!(env!("VERGEN_GIT_COMMIT_TIMESTAMP")), + let commit_date = match option_env!("COMMIT_DATE") { + Some("") | None => env!("VERGEN_GIT_COMMIT_TIMESTAMP"), Some(commit_date) => commit_date, }; From 78308365ecf40fb8b5416b43d2f7cee4cb8e33ac Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 29 Jul 2021 14:40:41 +0200 Subject: [PATCH 37/53] fix typos --- meilisearch-http/src/lib.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 89a578087..9df131365 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -3,15 +3,15 @@ //! //! To help you in this task, we'll try to do a little overview of the project. //! ## Milli -//! [Milli](https://github.com/meilisearch/milli) is the core library of MeiliSearch, it's where we actually index documents and perform searches. It's aim is to do theses two tasks as fast as possible. You can give an update to milli and it'll uses as many cores as provided to perform it as fast as possible. Nothing more. You can perform searches at the same time (search only use one core). -//! As you can see we're missing quite a lot of features here, milli does not handle multiples indexes, it can't queue updates, it doesn't provide any web / API frontend, it doesn't implements dumps or snapshots, etc... +//! [Milli](https://github.com/meilisearch/milli) is the core library of MeiliSearch. It's where we actually index documents and perform searches. Its purpose is to do these two tasks as fast as possible. You can give an update to milli, and it'll uses as many cores as provided to perform it as fast as possible. Nothing more. You can perform searches at the same time (search only uses one core). +//! As you can see, we're missing quite a lot of features here; milli does not handle multiples indexes, it can't queue updates, it doesn't provide any web / API frontend, it doesn't implement dumps or snapshots, etc... //! //! ## `Index` module -//! The [index] module is what encapsulate one milli index. It abstracts over its transaction and isolate a task that can be run into a thread. This is the unit of interaction with milli. -//! If you add a feature to milli, you'll probably need to add it in this module too before being able to expose it to the rest of meilisearch. +//! The [index] module is what encapsulates one milli index. It abstracts over its transaction and isolates a task that can be run into a thread. This is the unit of interaction with milli. +//! If you add a feature to milli, you'll probably need to add it in this module too before exposing it to the rest of meilisearch. //! //! ## `IndexController` module -//! To handle multiple indexes we created an [index_controller]. It's in charge of creating new indexes, keeping references to all its indexes forward asynchronous updates to its indexes and provide an API to synchronously search in its indexes. +//! To handle multiple indexes, we created an [index_controller]. It's in charge of creating new indexes, keeping references to all its indexes, forward asynchronous updates to its indexes, and provide an API to search in its indexes synchronously. //! To achieves this goal, we use an [actor model](https://en.wikipedia.org/wiki/Actor_model). //! //! ### The actor model @@ -30,12 +30,12 @@ //! - [`dump_actor`](index_controller/dump_actor/index.html) this actor handle the [dumps](https://docs.meilisearch.com/reference/api/dump.html). It needs to contact all the others actors and create a dump of everything that was currently happening. //! //! ## Data module -//! The [data] module provide an unified interface to communicate with the index controller and other services (snapshot, dumps, ...), initialize the MeiliSearch instance +//! The [data] module provide a unified interface to communicate with the index controller and other services (snapshot, dumps, ...), initialize the MeiliSearch instance //! -//! ## Http server +//! ## HTTP server //! To handle the web and API part, we are using [actix-web](https://docs.rs/actix-web/); you can find all routes in the [routes] module. -//! Currently the configuration of actix-web is made in the [lib.rs](crate). -//! Most of the routes uses [extractors] to handle the authentication. +//! Currently, the configuration of actix-web is made in the [lib.rs](crate). +//! Most of the routes use [extractors] to handle the authentication. #![allow(rustdoc::private_intra_doc_links)] pub mod data; From 9a8629a6a912955dc4ec37e09ea656561a186c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 29 Jul 2021 17:44:17 +0200 Subject: [PATCH 38/53] Update milli --- Cargo.lock | 4 ++-- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1e2ac465..9b4b28de1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1756,8 +1756,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.8.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.8.0#200e98c211510750e2ead475b8571b533ac6caa1" +version = "0.8.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.8.1#1290edd58a50d368106674cca28775b75e7ef183" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 6191ca2bb..4315326a4 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.8.0" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.8.1" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" From 53b4b2fcbc345df7f7e20eb7fbcbed39e341f336 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 29 Jul 2021 16:07:04 +0200 Subject: [PATCH 39/53] Use serdeval for validating json format. --- Cargo.lock | 17 ++++++++++------- meilisearch-http/Cargo.toml | 2 +- .../src/index_controller/update_actor/actor.rs | 14 +++++--------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1e2ac465..c046c31ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1666,7 +1666,6 @@ dependencies = [ "num_cpus", "obkv", "once_cell", - "oxidized-json-checker", "parking_lot", "paste", "pin-project", @@ -1679,6 +1678,7 @@ dependencies = [ "serde", "serde_json", "serde_url_params", + "serdeval", "sha-1 0.9.6", "sha2", "siphasher", @@ -1969,12 +1969,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "oxidized-json-checker" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938464aebf563f48ab86d1cfc0e2df952985c0b814d3108f41d1b85e7f5b0dac" - [[package]] name = "page_size" version = "0.4.2" @@ -2784,6 +2778,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serdeval" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94023adfd3d548a8bd9a1f09c09f44eaab7080c7a9ab20314bb65154bee62bd0" +dependencies = [ + "serde", +] + [[package]] name = "sha-1" version = "0.8.2" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 6191ca2bb..e251511ab 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -55,7 +55,6 @@ milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.8.0" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" -oxidized-json-checker = "0.3.2" parking_lot = "0.11.1" rand = "0.7.3" rayon = "1.5.0" @@ -77,6 +76,7 @@ obkv = "0.2.0" pin-project = "1.0.7" whoami = { version = "1.1.2", optional = true } reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true } +serdeval = "0.1.0" [dependencies.sentry] default-features = false diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index 8ba96dad1..59a22910f 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use async_stream::stream; use futures::StreamExt; use log::trace; -use oxidized_json_checker::JsonChecker; +use serdeval::*; use tokio::fs; use tokio::io::AsyncWriteExt; use tokio::sync::mpsc; @@ -180,7 +180,7 @@ where let update_store = self.store.clone(); tokio::task::spawn_blocking(move || { - use std::io::{copy, sink, BufReader, Seek}; + use std::io::{BufReader, Seek}; // If the payload is empty, ignore the check. let update_uuid = if let Some((mut file, uuid)) = file_path { @@ -188,14 +188,10 @@ where file.seek(SeekFrom::Start(0))?; // Check that the json payload is valid: let reader = BufReader::new(&mut file); - let mut checker = JsonChecker::new(reader); + // Validate that the payload is in the correct format. + let _: Seq> = serde_json::from_reader(reader) + .map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?; - if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() { - // The json file is invalid, we use Serde to get a nice error message: - file.seek(SeekFrom::Start(0))?; - let _: serde_json::Value = serde_json::from_reader(file) - .map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?; - } Some(uuid) } else { None From bf76d4a43c0d3e0816e7aa46db843694b7717718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 29 Jul 2021 18:14:36 +0200 Subject: [PATCH 40/53] Make clippy happy --- meilisearch-http/src/index/dump.rs | 2 +- meilisearch-http/src/index/mod.rs | 16 ++++++++-------- meilisearch-http/src/index/search.rs | 8 ++++---- meilisearch-http/src/index/updates.rs | 2 +- .../index_controller/dump_actor/loaders/v1.rs | 2 +- .../src/index_controller/index_actor/mod.rs | 6 +++--- .../index_controller/update_actor/store/dump.rs | 4 ++-- .../index_controller/update_actor/store/mod.rs | 4 ++-- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 263e3bd52..3b10a1562 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -101,7 +101,7 @@ impl Index { let index = Self::open(&dst_dir_path, size)?; let mut txn = index.write_txn()?; - let handler = UpdateHandler::new(&indexing_options)?; + let handler = UpdateHandler::new(indexing_options)?; index.update_settings_txn(&mut txn, &settings, handler.update_builder(0))?; diff --git a/meilisearch-http/src/index/mod.rs b/meilisearch-http/src/index/mod.rs index 4c963f1f5..37fd6fc39 100644 --- a/meilisearch-http/src/index/mod.rs +++ b/meilisearch-http/src/index/mod.rs @@ -62,34 +62,34 @@ impl Index { pub fn settings_txn(&self, txn: &RoTxn) -> Result> { let displayed_attributes = self - .displayed_fields(&txn)? + .displayed_fields(txn)? .map(|fields| fields.into_iter().map(String::from).collect()); let searchable_attributes = self - .searchable_fields(&txn)? + .searchable_fields(txn)? .map(|fields| fields.into_iter().map(String::from).collect()); - let filterable_attributes = self.filterable_fields(&txn)?.into_iter().collect(); + let filterable_attributes = self.filterable_fields(txn)?.into_iter().collect(); let criteria = self - .criteria(&txn)? + .criteria(txn)? .into_iter() .map(|c| c.to_string()) .collect(); let stop_words = self - .stop_words(&txn)? + .stop_words(txn)? .map(|stop_words| -> Result> { Ok(stop_words.stream().into_strs()?.into_iter().collect()) }) .transpose()? .unwrap_or_else(BTreeSet::new); - let distinct_field = self.distinct_field(&txn)?.map(String::from); + let distinct_field = self.distinct_field(txn)?.map(String::from); // in milli each word in the synonyms map were split on their separator. Since we lost // this information we are going to put space between words. let synonyms = self - .synonyms(&txn)? + .synonyms(txn)? .iter() .map(|(key, values)| { ( @@ -175,7 +175,7 @@ impl Index { attributes_to_retrieve: &Option>, fields_ids_map: &milli::FieldsIdsMap, ) -> Result> { - let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? { + let mut displayed_fields_ids = match self.displayed_fields_ids(txn)? { Some(ids) => ids.into_iter().collect::>(), None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index b2129ca53..620178773 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -239,7 +239,7 @@ fn compute_matches>( for (key, value) in document { let mut infos = Vec::new(); - compute_value_matches(&mut infos, value, matcher, &analyzer); + compute_value_matches(&mut infos, value, matcher, analyzer); if !infos.is_empty() { matches.insert(key.clone(), infos); } @@ -329,7 +329,7 @@ fn add_highlight_to_formatted_options( break; } - if let Some(id) = fields_ids_map.id(&attr) { + if let Some(id) = fields_ids_map.id(attr) { if displayed_ids.contains(&id) { formatted_options.insert(id, new_format); } @@ -366,7 +366,7 @@ fn add_crop_to_formatted_options( } } - if let Some(id) = fields_ids_map.id(&attr_name) { + if let Some(id) = fields_ids_map.id(attr_name) { if displayed_ids.contains(&id) { formatted_options .entry(id) @@ -592,7 +592,7 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> { // we highlight the complete word. None => { out.push_str(&self.marks.0); - out.push_str(&word); + out.push_str(word); out.push_str(&self.marks.1); } } diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index 09535721f..34fafd143 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -205,7 +205,7 @@ impl Index { // Set the primary key if not set already, ignore if already set. if let (None, Some(primary_key)) = (self.primary_key(txn)?, primary_key) { - let mut builder = UpdateBuilder::new(0).settings(txn, &self); + let mut builder = UpdateBuilder::new(0).settings(txn, self); builder.set_primary_key(primary_key.to_string()); builder.execute(|_, _| ())?; } diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index a7f1aa8d1..c631e113c 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -98,7 +98,7 @@ fn load_index( let mut txn = index.write_txn()?; - let handler = UpdateHandler::new(&indexer_options)?; + let handler = UpdateHandler::new(indexer_options)?; index.update_settings_txn(&mut txn, &settings.check(), handler.update_builder(0))?; diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index 4085dc0bd..faad75e01 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -40,9 +40,9 @@ impl IndexMeta { } fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result { - let created_at = index.created_at(&txn)?; - let updated_at = index.updated_at(&txn)?; - let primary_key = index.primary_key(&txn)?.map(String::from); + let created_at = index.created_at(txn)?; + let updated_at = index.updated_at(txn)?; + let primary_key = index.primary_key(txn)?.map(String::from); Ok(Self { created_at, updated_at, diff --git a/meilisearch-http/src/index_controller/update_actor/store/dump.rs b/meilisearch-http/src/index_controller/update_actor/store/dump.rs index 7c46f98fa..79a3cca05 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/dump.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/dump.rs @@ -59,8 +59,8 @@ impl UpdateStore { let update_files_path = path.as_ref().join(super::UPDATE_DIR); create_dir_all(&update_files_path)?; - self.dump_pending(&txn, uuids, &mut dump_data_file, &path)?; - self.dump_completed(&txn, uuids, &mut dump_data_file)?; + self.dump_pending(txn, uuids, &mut dump_data_file, &path)?; + self.dump_completed(txn, uuids, &mut dump_data_file)?; Ok(()) } diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 8e85429ca..613802b59 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -268,13 +268,13 @@ impl UpdateStore { self.pending_queue.remap_key_type::().put( wtxn, &(global_id, index_uuid, enqueued.id()), - &enqueued, + enqueued, )?; } _ => { let _update_id = self.next_update_id_raw(wtxn, index_uuid)?; self.updates - .put(wtxn, &(index_uuid, update.id()), &update)?; + .put(wtxn, &(index_uuid, update.id()), update)?; } } Ok(()) From 1b5d918cb94124caaf07ecd3bca9c8e929726c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 29 Jul 2021 18:32:09 +0200 Subject: [PATCH 41/53] Fix rustfmt --- .../src/index_controller/update_actor/store/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meilisearch-http/src/index_controller/update_actor/store/mod.rs b/meilisearch-http/src/index_controller/update_actor/store/mod.rs index 613802b59..e23e05b52 100644 --- a/meilisearch-http/src/index_controller/update_actor/store/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/store/mod.rs @@ -273,8 +273,7 @@ impl UpdateStore { } _ => { let _update_id = self.next_update_id_raw(wtxn, index_uuid)?; - self.updates - .put(wtxn, &(index_uuid, update.id()), update)?; + self.updates.put(wtxn, &(index_uuid, update.id()), update)?; } } Ok(()) From 31fe263356c7f070dc7ac6782b80c4572ba9bb43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 5 Aug 2021 13:08:27 +0200 Subject: [PATCH 42/53] Update milli to v0.9.0 --- Cargo.lock | 4 ++-- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9ea7184c..4c9df2ba4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1756,8 +1756,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.8.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.8.1#1290edd58a50d368106674cca28775b75e7ef183" +version = "0.9.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.9.0#2102e0da6b046f252b75b2bb5a9fd34a5954ac8b" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 0e34474a7..e03ca7285 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.8.1" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.9.0" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" From 992b082c6f3b44cc4522e1a6c11681f6e7c7f8cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 5 Aug 2021 13:28:32 +0200 Subject: [PATCH 43/53] Fix docs link anchor --- meilisearch-http/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 33c99191e..54e2f5dfc 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -145,7 +145,7 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { " Thank you for using MeiliSearch! -We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics +We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#disable-analytics Anonymous telemetry: \"Enabled\"" ); From 3084537d1ed03625b767ad792a266de0f8c616e1 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 4 Aug 2021 10:33:30 +0200 Subject: [PATCH 44/53] restore attributes for faceting in dump v1 --- .../src/index_controller/dump_actor/loaders/v1.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index c631e113c..c0c7b8ef2 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -73,7 +73,7 @@ struct Settings { #[serde(default, deserialize_with = "deserialize_some")] pub synonyms: Option>>>, #[serde(default, deserialize_with = "deserialize_some")] - pub filterable_attributes: Option>>, + pub attributes_for_faceting: Option>>, } fn load_index( @@ -142,7 +142,7 @@ impl From for index_controller::Settings { // representing the name of the faceted field + the type of the field. Since the type // was not known in the V1 of the dump we are just going to assume everything is a // String - filterable_attributes: settings.filterable_attributes.map(|o| o.map(|vec| vec.into_iter().collect())), + filterable_attributes: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().collect())), // we need to convert the old `Vec` into a `BTreeSet` ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { match criterion.as_str() { From 29ca8271b36d73e061fdebbb271a8e38d6ca0063 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 4 Aug 2021 11:02:17 +0200 Subject: [PATCH 45/53] test dumpv1 format regression --- .../src/index_controller/dump_actor/loaders/v1.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index c0c7b8ef2..c19ba4329 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -180,3 +180,17 @@ fn import_settings(dir_path: impl AsRef) -> anyhow::Result { Ok(metadata) } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn settings_format_regression() { + let settings = Settings::default(); + assert_eq!( + r##"{"rankingRules":null,"distinctAttribute":null,"searchableAttributes":null,"displayedAttributes":null,"stopWords":null,"synonyms":null,"attributesForFaceting":null}"##, + serde_json::to_string(&settings).unwrap() + ); + } +} From f5ddea481aebcfc966760b6fcf4bb89e3979796b Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 4 Aug 2021 14:40:08 +0200 Subject: [PATCH 46/53] reintroduce exactness --- .../src/index_controller/dump_actor/loaders/v1.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index c19ba4329..034e464b1 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -146,14 +146,10 @@ impl From for index_controller::Settings { // we need to convert the old `Vec` into a `BTreeSet` ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { match criterion.as_str() { - "words" | "typo" | "proximity" | "attribute" => Some(criterion), + "words" | "typo" | "proximity" | "attribute" | "exactness" => Some(criterion), s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), "wordsPosition" => { - warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored"); - Some(String::from("words")) - } - "exactness" => { - error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion); + warn!("The criteria `attribute` and `wordsPosition` have been merged into a single criterion `attribute` so `wordsPositon` will be ignored"); None } s => { From 8f05d8d546d56631e4ec748a4c24e71f870d778f Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 5 Aug 2021 15:43:48 +0200 Subject: [PATCH 47/53] fix clippy warnings --- Cargo.lock | 2 +- .../src/index_controller/dump_actor/loaders/v1.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9ea7184c..c1adeeabc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,7 +1700,7 @@ dependencies = [ [[package]] name = "meilisearch-tokenizer" version = "0.2.3" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781" +source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781" dependencies = [ "character_converter", "cow-utils", diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index 034e464b1..30923584e 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -144,17 +144,17 @@ impl From for index_controller::Settings { // String filterable_attributes: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().collect())), // we need to convert the old `Vec` into a `BTreeSet` - ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| { + ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter(|criterion| { match criterion.as_str() { - "words" | "typo" | "proximity" | "attribute" | "exactness" => Some(criterion), - s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion), + "words" | "typo" | "proximity" | "attribute" | "exactness" => true, + s if s.starts_with("asc") || s.starts_with("desc") => true, "wordsPosition" => { warn!("The criteria `attribute` and `wordsPosition` have been merged into a single criterion `attribute` so `wordsPositon` will be ignored"); - None + false } s => { error!("Unknown criterion found in the dump: `{}`, it will be ignored", s); - None + false } } }).collect())), From 75312807646ccae74630a15c07d90d5976108315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 12 Aug 2021 13:55:47 +0200 Subject: [PATCH 48/53] Update tokenizer version to v0.2.4 --- Cargo.lock | 31 +++---------------------------- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53625e717..260d3a0ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1658,7 +1658,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer 0.2.3", + "meilisearch-tokenizer", "memmap", "milli", "mime", @@ -1697,22 +1697,6 @@ dependencies = [ "zip", ] -[[package]] -name = "meilisearch-tokenizer" -version = "0.2.3" -source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781" -dependencies = [ - "character_converter", - "cow-utils", - "deunicode", - "fst", - "jieba-rs", - "once_cell", - "slice-group-by", - "unicode-segmentation", - "whatlang 0.9.0", -] - [[package]] name = "meilisearch-tokenizer" version = "0.2.4" @@ -1726,7 +1710,7 @@ dependencies = [ "once_cell", "slice-group-by", "unicode-segmentation", - "whatlang 0.12.0", + "whatlang", ] [[package]] @@ -1776,7 +1760,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer 0.2.4", + "meilisearch-tokenizer", "memmap", "obkv", "once_cell", @@ -3554,15 +3538,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "whatlang" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075" -dependencies = [ - "hashbrown 0.7.2", -] - [[package]] name = "whatlang" version = "0.12.0" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e03ca7285..d76d5052a 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -49,7 +49,7 @@ itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.4" } memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.9.0" } mime = "0.3.16" From 621529e9dc0d7646b61b1134e9f07176b452ad72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 12 Aug 2021 18:58:07 +0200 Subject: [PATCH 49/53] Update telemetry link --- meilisearch-http/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 54e2f5dfc..78dacd640 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -145,7 +145,7 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { " Thank you for using MeiliSearch! -We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#disable-analytics +We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html Anonymous telemetry: \"Enabled\"" ); From f3e595e2f0ee37ab3c4a68b65c669bdc0e42035a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Thu, 12 Aug 2021 13:28:44 +0200 Subject: [PATCH 50/53] Update milli dependency --- Cargo.lock | 4 ++-- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53625e717..e741e54ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1756,8 +1756,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.9.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.9.0#2102e0da6b046f252b75b2bb5a9fd34a5954ac8b" +version = "0.10.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.10.0#51581d14f859b01411b586475f48b54eb81b7b1d" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index e03ca7285..bd10c71f2 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.9.0" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.10.0" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" From 39d5a990953ab0095710e7ba72859b87e29d9529 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 18 Aug 2021 12:09:34 +0200 Subject: [PATCH 51/53] Update milli and tokenizer version --- Cargo.lock | 6 +++--- meilisearch-http/Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0ad42a49..94d9caeee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1699,8 +1699,8 @@ dependencies = [ [[package]] name = "meilisearch-tokenizer" -version = "0.2.4" -source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.4#135d08dce465a756abaf6a1bcad70f315bda99b9" +version = "0.2.5" +source = "git+https://github.com/meilisearch/tokenizer.git?branch=main#9a2ff20a2b6006f966fe9ee9df56da5108ec7d5e" dependencies = [ "character_converter", "cow-utils", @@ -1741,7 +1741,7 @@ dependencies = [ [[package]] name = "milli" version = "0.10.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.10.0#51581d14f859b01411b586475f48b54eb81b7b1d" +source = "git+https://github.com/meilisearch/milli.git?branch=update-tokenizer-0.2.5#e938ea07f62286ba72b18364941be3ec2f727d87" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 58141a159..0d5e50aec 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -49,9 +49,9 @@ itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.4" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.10.0" } +milli = { git = "https://github.com/meilisearch/milli.git", branch = "update-tokenizer-0.2.5" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" From 38fc8767048d777cfddb166da1875d35c004e740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Wed, 18 Aug 2021 14:55:10 +0200 Subject: [PATCH 52/53] Update tokenizer and new milli version with new tags --- Cargo.lock | 6 +++--- meilisearch-http/Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94d9caeee..79bd2e948 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,7 +1700,7 @@ dependencies = [ [[package]] name = "meilisearch-tokenizer" version = "0.2.5" -source = "git+https://github.com/meilisearch/tokenizer.git?branch=main#9a2ff20a2b6006f966fe9ee9df56da5108ec7d5e" +source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.5#c0b5cf741ed9485147f2cbe523f2214d4fa4c395" dependencies = [ "character_converter", "cow-utils", @@ -1740,8 +1740,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.10.0" -source = "git+https://github.com/meilisearch/milli.git?branch=update-tokenizer-0.2.5#e938ea07f62286ba72b18364941be3ec2f727d87" +version = "0.10.1" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.10.1#198c416bd8dd1813d5e6efe37cc0e6273185a112" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 0d5e50aec..a140d8793 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -49,9 +49,9 @@ itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", branch = "main" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", branch = "update-tokenizer-0.2.5" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.10.1" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2" From 149f46c18465dc65a3905360a5c5598b7c06a85e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Mon, 23 Aug 2021 12:07:28 +0200 Subject: [PATCH 53/53] Fix pacic when decoding --- Cargo.lock | 4 ++-- meilisearch-http/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79bd2e948..43222363e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1740,8 +1740,8 @@ dependencies = [ [[package]] name = "milli" -version = "0.10.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.10.1#198c416bd8dd1813d5e6efe37cc0e6273185a112" +version = "0.10.2" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.10.2#879d5e8799836d93f8995810965b6797be4f69d1" dependencies = [ "bstr", "byteorder", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index a140d8793..0f8a5e21f 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -51,7 +51,7 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.10.1" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.10.2" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.5.2"