diff --git a/Cargo.lock b/Cargo.lock index b26636a01..2ba905455 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2038,6 +2038,7 @@ name = "meilisearch-http" version = "0.29.1" dependencies = [ "actix-cors", + "actix-http", "actix-rt", "actix-web", "actix-web-static-files", @@ -2045,6 +2046,7 @@ dependencies = [ "assert-json-diff", "async-stream", "async-trait", + "brotli", "bstr 1.0.1", "byte-unit", "bytes", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index cabb6ed48..12eca84fc 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -24,6 +24,7 @@ zip = { version = "0.6.2", optional = true } [dependencies] actix-cors = "0.6.3" actix-web = { version = "4.2.1", default-features = false, features = ["macros", "compress-brotli", "compress-gzip", "cookies", "rustls"] } +actix-http = "3.2.2" actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true } anyhow = { version = "1.0.65", features = ["backtrace"] } async-stream = "0.3.3" @@ -85,6 +86,7 @@ lazy_static = "1.4.0" [dev-dependencies] actix-rt = "2.7.0" assert-json-diff = "2.0.2" +brotli = "3.3.4" manifest-dir-macros = "0.1.16" maplit = "1.0.2" urlencoding = "2.1.2" diff --git a/meilisearch-http/src/extractors/payload.rs b/meilisearch-http/src/extractors/payload.rs index 6cd8df190..f16fdd67b 100644 --- a/meilisearch-http/src/extractors/payload.rs +++ b/meilisearch-http/src/extractors/payload.rs @@ -1,13 +1,14 @@ use std::pin::Pin; use std::task::{Context, Poll}; +use actix_http::encoding::Decoder as Decompress; use actix_web::error::PayloadError; use actix_web::{dev, web, FromRequest, HttpRequest}; use futures::future::{ready, Ready}; use futures::Stream; pub struct Payload { - payload: dev::Payload, + payload: Decompress, limit: usize, } @@ -39,7 +40,7 @@ impl FromRequest for Payload { .map(|c| c.limit) .unwrap_or(PayloadConfig::default().limit); ready(Ok(Payload { - payload: payload.take(), + payload: Decompress::from_headers(payload.take(), req.headers()), limit, })) } diff --git a/meilisearch-http/tests/common/encoder.rs b/meilisearch-http/tests/common/encoder.rs new file mode 100644 index 000000000..2363ec4f9 --- /dev/null +++ b/meilisearch-http/tests/common/encoder.rs @@ -0,0 +1,87 @@ +use actix_http::header::TryIntoHeaderPair; +use bytes::Bytes; +use flate2::read::{GzDecoder, ZlibDecoder}; +use flate2::write::{GzEncoder, ZlibEncoder}; +use flate2::Compression; +use std::io::{Read, Write}; + +#[derive(Clone, Copy)] +pub enum Encoder { + Plain, + Gzip, + Deflate, + Brotli, +} + +impl Encoder { + pub fn encode(self: &Encoder, body: impl Into) -> impl Into { + match self { + Self::Gzip => { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(&body.into()) + .expect("Failed to encode request body"); + encoder.finish().expect("Failed to encode request body") + } + Self::Deflate => { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(&body.into()) + .expect("Failed to encode request body"); + encoder.finish().unwrap() + } + Self::Plain => Vec::from(body.into()), + Self::Brotli => { + let mut encoder = brotli::CompressorWriter::new(Vec::new(), 32 * 1024, 3, 22); + encoder + .write_all(&body.into()) + .expect("Failed to encode request body"); + encoder.flush().expect("Failed to encode request body"); + encoder.into_inner() + } + } + } + + pub fn decode(self: &Encoder, bytes: impl Into) -> impl Into { + let mut buffer = Vec::new(); + let input = bytes.into(); + match self { + Self::Gzip => { + GzDecoder::new(input.as_ref()) + .read_to_end(&mut buffer) + .expect("Invalid gzip stream"); + } + Self::Deflate => { + ZlibDecoder::new(input.as_ref()) + .read_to_end(&mut buffer) + .expect("Invalid zlib stream"); + } + Self::Plain => { + buffer + .write_all(input.as_ref()) + .expect("Unexpected memory copying issue"); + } + Self::Brotli => { + brotli::Decompressor::new(input.as_ref(), 4096) + .read_to_end(&mut buffer) + .expect("Invalid brotli stream"); + } + }; + buffer + } + + pub fn header(self: &Encoder) -> Option { + match self { + Self::Plain => None, + Self::Gzip => Some(("Content-Encoding", "gzip")), + Self::Deflate => Some(("Content-Encoding", "deflate")), + Self::Brotli => Some(("Content-Encoding", "br")), + } + } + + pub fn iterator() -> impl Iterator { + [Self::Plain, Self::Gzip, Self::Deflate, Self::Brotli] + .iter() + .copied() + } +} diff --git a/meilisearch-http/tests/common/index.rs b/meilisearch-http/tests/common/index.rs index 90d138ced..43534074d 100644 --- a/meilisearch-http/tests/common/index.rs +++ b/meilisearch-http/tests/common/index.rs @@ -7,24 +7,27 @@ use std::{ use actix_web::http::StatusCode; use serde_json::{json, Value}; use tokio::time::sleep; -use urlencoding::encode; +use urlencoding::encode as urlencode; use super::service::Service; +use super::encoder::Encoder; + pub struct Index<'a> { pub uid: String, pub service: &'a Service, + pub encoder: Encoder, } #[allow(dead_code)] impl Index<'_> { pub async fn get(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", encode(self.uid.as_ref())); + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); self.service.get(url).await } pub async fn load_test_set(&self) -> u64 { - let url = format!("/indexes/{}/documents", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref())); let (response, code) = self .service .post_str(url, include_str!("../assets/test_set.json")) @@ -40,20 +43,22 @@ impl Index<'_> { "uid": self.uid, "primaryKey": primary_key, }); - self.service.post("/indexes", body).await + self.service + .post_encoded("/indexes", body, self.encoder) + .await } pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) { let body = json!({ "primaryKey": primary_key, }); - let url = format!("/indexes/{}", encode(self.uid.as_ref())); + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); - self.service.patch(url, body).await + self.service.patch_encoded(url, body, self.encoder).await } pub async fn delete(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", encode(self.uid.as_ref())); + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); self.service.delete(url).await } @@ -65,12 +70,14 @@ impl Index<'_> { let url = match primary_key { Some(key) => format!( "/indexes/{}/documents?primaryKey={}", - encode(self.uid.as_ref()), + urlencode(self.uid.as_ref()), key ), - None => format!("/indexes/{}/documents", encode(self.uid.as_ref())), + None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())), }; - self.service.post(url, documents).await + self.service + .post_encoded(url, documents, self.encoder) + .await } pub async fn update_documents( @@ -81,12 +88,12 @@ impl Index<'_> { let url = match primary_key { Some(key) => format!( "/indexes/{}/documents?primaryKey={}", - encode(self.uid.as_ref()), + urlencode(self.uid.as_ref()), key ), - None => format!("/indexes/{}/documents", encode(self.uid.as_ref())), + None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())), }; - self.service.put(url, documents).await + self.service.put_encoded(url, documents, self.encoder).await } pub async fn wait_task(&self, update_id: u64) -> Value { @@ -132,7 +139,7 @@ impl Index<'_> { id: u64, options: Option, ) -> (Value, StatusCode) { - let mut url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); + let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id); if let Some(fields) = options.and_then(|o| o.fields) { let _ = write!(url, "?fields={}", fields.join(",")); } @@ -140,7 +147,7 @@ impl Index<'_> { } pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) { - let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref())); + let mut url = format!("/indexes/{}/documents?", urlencode(self.uid.as_ref())); if let Some(limit) = options.limit { let _ = write!(url, "limit={}&", limit); } @@ -157,42 +164,44 @@ impl Index<'_> { } pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); + let url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id); self.service.delete(url).await } pub async fn clear_all_documents(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref())); self.service.delete(url).await } pub async fn delete_batch(&self, ids: Vec) -> (Value, StatusCode) { let url = format!( "/indexes/{}/documents/delete-batch", - encode(self.uid.as_ref()) + urlencode(self.uid.as_ref()) ); self.service - .post(url, serde_json::to_value(&ids).unwrap()) + .post_encoded(url, serde_json::to_value(&ids).unwrap(), self.encoder) .await } pub async fn settings(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref())); self.service.get(url).await } pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); - self.service.patch(url, settings).await + let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref())); + self.service + .patch_encoded(url, settings, self.encoder) + .await } pub async fn delete_settings(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref())); self.service.delete(url).await } pub async fn stats(&self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/stats", encode(self.uid.as_ref())); + let url = format!("/indexes/{}/stats", urlencode(self.uid.as_ref())); self.service.get(url).await } @@ -217,29 +226,33 @@ impl Index<'_> { } pub async fn search_post(&self, query: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/search", encode(self.uid.as_ref())); - self.service.post(url, query).await + let url = format!("/indexes/{}/search", urlencode(self.uid.as_ref())); + self.service.post_encoded(url, query, self.encoder).await } pub async fn search_get(&self, query: Value) -> (Value, StatusCode) { let params = yaup::to_string(&query).unwrap(); - let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params); + let url = format!( + "/indexes/{}/search?{}", + urlencode(self.uid.as_ref()), + params + ); self.service.get(url).await } pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) { let url = format!( "/indexes/{}/settings/{}", - encode(self.uid.as_ref()), + urlencode(self.uid.as_ref()), "distinct-attribute" ); - self.service.put(url, value).await + self.service.put_encoded(url, value, self.encoder).await } pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) { let url = format!( "/indexes/{}/settings/{}", - encode(self.uid.as_ref()), + urlencode(self.uid.as_ref()), "distinct-attribute" ); self.service.get(url).await diff --git a/meilisearch-http/tests/common/mod.rs b/meilisearch-http/tests/common/mod.rs index b076b0ea5..c4793a0cb 100644 --- a/meilisearch-http/tests/common/mod.rs +++ b/meilisearch-http/tests/common/mod.rs @@ -1,3 +1,4 @@ +pub mod encoder; pub mod index; pub mod server; pub mod service; diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 30a64c90e..f243a11b9 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -12,6 +12,7 @@ use once_cell::sync::Lazy; use serde_json::Value; use tempfile::TempDir; +use crate::common::encoder::Encoder; use meilisearch_http::option::Opt; use super::index::Index; @@ -100,9 +101,14 @@ impl Server { /// Returns a view to an index. There is no guarantee that the index exists. pub fn index(&self, uid: impl AsRef) -> Index<'_> { + self.index_with_encoder(uid, Encoder::Plain) + } + + pub fn index_with_encoder(&self, uid: impl AsRef, encoder: Encoder) -> Index<'_> { Index { uid: uid.as_ref().to_string(), service: &self.service, + encoder, } } diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index 0834abf8d..3a3f6021c 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -1,8 +1,11 @@ +use actix_web::http::header::ContentType; +use actix_web::test::TestRequest; use actix_web::{http::StatusCode, test}; use meilisearch_auth::AuthController; use meilisearch_lib::MeiliSearch; use serde_json::Value; +use crate::common::encoder::Encoder; use meilisearch_http::{analytics, create_app, Opt}; pub struct Service { @@ -14,26 +17,18 @@ pub struct Service { impl Service { pub async fn post(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; + self.post_encoded(url, body, Encoder::Plain).await + } - let mut req = test::TestRequest::post().uri(url.as_ref()).set_json(&body); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + pub async fn post_encoded( + &self, + url: impl AsRef, + body: Value, + encoder: Encoder, + ) -> (Value, StatusCode) { + let mut req = test::TestRequest::post().uri(url.as_ref()); + req = self.encode(req, body, encoder); + self.request(req).await } /// Send a test post request from a text body, with a `content-type:application/json` header. @@ -42,101 +37,54 @@ impl Service { url: impl AsRef, body: impl AsRef, ) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; - - let mut req = test::TestRequest::post() + let req = test::TestRequest::post() .uri(url.as_ref()) .set_payload(body.as_ref().to_string()) .insert_header(("content-type", "application/json")); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + self.request(req).await } pub async fn get(&self, url: impl AsRef) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; - - let mut req = test::TestRequest::get().uri(url.as_ref()); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + let req = test::TestRequest::get().uri(url.as_ref()); + self.request(req).await } pub async fn put(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; + self.put_encoded(url, body, Encoder::Plain).await + } - let mut req = test::TestRequest::put().uri(url.as_ref()).set_json(&body); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + pub async fn put_encoded( + &self, + url: impl AsRef, + body: Value, + encoder: Encoder, + ) -> (Value, StatusCode) { + let mut req = test::TestRequest::put().uri(url.as_ref()); + req = self.encode(req, body, encoder); + self.request(req).await } pub async fn patch(&self, url: impl AsRef, body: Value) -> (Value, StatusCode) { - let app = test::init_service(create_app!( - &self.meilisearch, - &self.auth, - true, - self.options, - analytics::MockAnalytics::new(&self.options).0 - )) - .await; + self.patch_encoded(url, body, Encoder::Plain).await + } - let mut req = test::TestRequest::patch().uri(url.as_ref()).set_json(&body); - if let Some(api_key) = &self.api_key { - req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); - } - let req = req.to_request(); - let res = test::call_service(&app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) + pub async fn patch_encoded( + &self, + url: impl AsRef, + body: Value, + encoder: Encoder, + ) -> (Value, StatusCode) { + let mut req = test::TestRequest::patch().uri(url.as_ref()); + req = self.encode(req, body, encoder); + self.request(req).await } pub async fn delete(&self, url: impl AsRef) -> (Value, StatusCode) { + let req = test::TestRequest::delete().uri(url.as_ref()); + self.request(req).await + } + + pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) { let app = test::init_service(create_app!( &self.meilisearch, &self.auth, @@ -146,7 +94,6 @@ impl Service { )) .await; - let mut req = test::TestRequest::delete().uri(url.as_ref()); if let Some(api_key) = &self.api_key { req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); } @@ -158,4 +105,16 @@ impl Service { let response = serde_json::from_slice(&body).unwrap_or_default(); (response, status_code) } + + fn encode(&self, req: TestRequest, body: Value, encoder: Encoder) -> TestRequest { + let bytes = serde_json::to_string(&body).expect("Failed to serialize test data to json"); + let encoded_body = encoder.encode(bytes); + let header = encoder.header(); + match header { + Some(header) => req.insert_header(header), + None => req, + } + .set_payload(encoded_body) + .insert_header(ContentType::json()) + } } diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 8e6ba44a9..48ef6276b 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -1,6 +1,7 @@ use crate::common::{GetAllDocumentsOptions, Server}; use actix_web::test; +use crate::common::encoder::Encoder; use meilisearch_http::{analytics, create_app}; use serde_json::{json, Value}; use time::{format_description::well_known::Rfc3339, OffsetDateTime}; @@ -97,6 +98,95 @@ async fn add_single_document_test_json_content_types() { assert_eq!(response["taskUid"], 1); } +/// Here we try sending encoded (compressed) document request +#[actix_rt::test] +async fn add_single_document_gzip_encoded() { + let document = json!({ + "id": 1, + "content": "Bouvier Bernois", + }); + + // this is a what is expected and should work + let server = Server::new().await; + let app = test::init_service(create_app!( + &server.service.meilisearch, + &server.service.auth, + true, + server.service.options, + analytics::MockAnalytics::new(&server.service.options).0 + )) + .await; + // post + let document = serde_json::to_string(&document).unwrap(); + let encoder = Encoder::Gzip; + let req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(encoder.encode(document.clone())) + .insert_header(("content-type", "application/json")) + .insert_header(encoder.header().unwrap()) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response["taskUid"], 0); + + // put + let req = test::TestRequest::put() + .uri("/indexes/dog/documents") + .set_payload(encoder.encode(document)) + .insert_header(("content-type", "application/json")) + .insert_header(encoder.header().unwrap()) + .to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response["taskUid"], 1); +} + +/// Here we try document request with every encoding +#[actix_rt::test] +async fn add_single_document_with_every_encoding() { + let document = json!({ + "id": 1, + "content": "Bouvier Bernois", + }); + + // this is a what is expected and should work + let server = Server::new().await; + let app = test::init_service(create_app!( + &server.service.meilisearch, + &server.service.auth, + true, + server.service.options, + analytics::MockAnalytics::new(&server.service.options).0 + )) + .await; + // post + let document = serde_json::to_string(&document).unwrap(); + + for (task_uid, encoder) in Encoder::iterator().enumerate() { + let mut req = test::TestRequest::post() + .uri("/indexes/dog/documents") + .set_payload(encoder.encode(document.clone())) + .insert_header(("content-type", "application/json")); + req = match encoder.header() { + Some(header) => req.insert_header(header), + None => req, + }; + let req = req.to_request(); + let res = test::call_service(&app, req).await; + let status_code = res.status(); + let body = test::read_body(res).await; + let response: Value = serde_json::from_slice(&body).unwrap_or_default(); + assert_eq!(status_code, 202); + assert_eq!(response["taskUid"], task_uid); + } +} + /// any other content-type is must be refused #[actix_rt::test] async fn error_add_documents_test_bad_content_types() { @@ -690,23 +780,6 @@ async fn error_document_add_create_index_bad_uid() { assert_eq!(response, expected_response); } -#[actix_rt::test] -async fn error_document_update_create_index_bad_uid() { - let server = Server::new().await; - let index = server.index("883 fj!"); - let (response, code) = index.update_documents(json!([{"id": 1}]), None).await; - - let expected_response = json!({ - "message": "invalid index uid `883 fj!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.", - "code": "invalid_index_uid", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_index_uid" - }); - - assert_eq!(code, 400); - assert_eq!(response, expected_response); -} - #[actix_rt::test] async fn document_addition_with_primary_key() { let server = Server::new().await; @@ -736,35 +809,6 @@ async fn document_addition_with_primary_key() { assert_eq!(response["primaryKey"], "primary"); } -#[actix_rt::test] -async fn document_update_with_primary_key() { - let server = Server::new().await; - let index = server.index("test"); - - let documents = json!([ - { - "primary": 1, - "content": "foo", - } - ]); - let (_response, code) = index.update_documents(documents, Some("primary")).await; - assert_eq!(code, 202); - - index.wait_task(0).await; - - let (response, code) = index.get_task(0).await; - assert_eq!(code, 200); - assert_eq!(response["status"], "succeeded"); - assert_eq!(response["uid"], 0); - assert_eq!(response["type"], "documentAdditionOrUpdate"); - assert_eq!(response["details"]["indexedDocuments"], 1); - assert_eq!(response["details"]["receivedDocuments"], 1); - - let (response, code) = index.get().await; - assert_eq!(code, 200); - assert_eq!(response["primaryKey"], "primary"); -} - #[actix_rt::test] async fn replace_document() { let server = Server::new().await; @@ -811,47 +855,6 @@ async fn add_no_documents() { assert_eq!(code, 202); } -#[actix_rt::test] -async fn update_document() { - let server = Server::new().await; - let index = server.index("test"); - - let documents = json!([ - { - "doc_id": 1, - "content": "foo", - } - ]); - - let (_response, code) = index.add_documents(documents, None).await; - assert_eq!(code, 202); - - index.wait_task(0).await; - - let documents = json!([ - { - "doc_id": 1, - "other": "bar", - } - ]); - - let (response, code) = index.update_documents(documents, None).await; - assert_eq!(code, 202, "response: {}", response); - - index.wait_task(1).await; - - let (response, code) = index.get_task(1).await; - assert_eq!(code, 200); - assert_eq!(response["status"], "succeeded"); - - let (response, code) = index.get_document(1, None).await; - assert_eq!(code, 200); - assert_eq!( - response.to_string(), - r##"{"doc_id":1,"content":"foo","other":"bar"}"## - ); -} - #[actix_rt::test] async fn add_larger_dataset() { let server = Server::new().await; @@ -873,27 +876,6 @@ async fn add_larger_dataset() { assert_eq!(response["results"].as_array().unwrap().len(), 77); } -#[actix_rt::test] -async fn update_larger_dataset() { - let server = Server::new().await; - let index = server.index("test"); - let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(); - index.update_documents(documents, None).await; - index.wait_task(0).await; - let (response, code) = index.get_task(0).await; - assert_eq!(code, 200); - assert_eq!(response["type"], "documentAdditionOrUpdate"); - assert_eq!(response["details"]["indexedDocuments"], 77); - let (response, code) = index - .get_all_documents(GetAllDocumentsOptions { - limit: Some(1000), - ..Default::default() - }) - .await; - assert_eq!(code, 200); - assert_eq!(response["results"].as_array().unwrap().len(), 77); -} - #[actix_rt::test] async fn error_add_documents_bad_document_id() { let server = Server::new().await; @@ -924,34 +906,6 @@ async fn error_add_documents_bad_document_id() { ); } -#[actix_rt::test] -async fn error_update_documents_bad_document_id() { - let server = Server::new().await; - let index = server.index("test"); - index.create(Some("docid")).await; - let documents = json!([ - { - "docid": "foo & bar", - "content": "foobar" - } - ]); - index.update_documents(documents, None).await; - let response = index.wait_task(1).await; - assert_eq!(response["status"], json!("failed")); - assert_eq!( - response["error"]["message"], - json!( - r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."# - ) - ); - assert_eq!(response["error"]["code"], json!("invalid_document_id")); - assert_eq!(response["error"]["type"], json!("invalid_request")); - assert_eq!( - response["error"]["link"], - json!("https://docs.meilisearch.com/errors#invalid_document_id") - ); -} - #[actix_rt::test] async fn error_add_documents_missing_document_id() { let server = Server::new().await; @@ -980,32 +934,6 @@ async fn error_add_documents_missing_document_id() { ); } -#[actix_rt::test] -async fn error_update_documents_missing_document_id() { - let server = Server::new().await; - let index = server.index("test"); - index.create(Some("docid")).await; - let documents = json!([ - { - "id": "11", - "content": "foobar" - } - ]); - index.update_documents(documents, None).await; - let response = index.wait_task(1).await; - assert_eq!(response["status"], "failed"); - assert_eq!( - response["error"]["message"], - r#"Document doesn't have a `docid` attribute: `{"id":"11","content":"foobar"}`."# - ); - assert_eq!(response["error"]["code"], "missing_document_id"); - assert_eq!(response["error"]["type"], "invalid_request"); - assert_eq!( - response["error"]["link"], - "https://docs.meilisearch.com/errors#missing_document_id" - ); -} - #[actix_rt::test] #[ignore] // // TODO: Fix in an other PR: this does not provoke any error. async fn error_document_field_limit_reached() { diff --git a/meilisearch-http/tests/documents/get_documents.rs b/meilisearch-http/tests/documents/get_documents.rs index c15d3f7fa..f3a25e720 100644 --- a/meilisearch-http/tests/documents/get_documents.rs +++ b/meilisearch-http/tests/documents/get_documents.rs @@ -1,6 +1,11 @@ use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server}; +use actix_web::test; +use http::header::ACCEPT_ENCODING; -use serde_json::json; +use crate::common::encoder::Encoder; +use meilisearch_http::{analytics, create_app}; +use serde_json::{json, Value}; +use urlencoding::encode as urlencode; // TODO: partial test since we are testing error, amd error is not yet fully implemented in // transplant @@ -155,6 +160,40 @@ async fn get_all_documents_no_options() { assert_eq!(first, arr[0]); } +#[actix_rt::test] +async fn get_all_documents_no_options_with_response_compression() { + let server = Server::new().await; + let index_uid = "test"; + let index = server.index(index_uid); + index.load_test_set().await; + + let app = test::init_service(create_app!( + &server.service.meilisearch, + &server.service.auth, + true, + server.service.options, + analytics::MockAnalytics::new(&server.service.options).0 + )) + .await; + + let req = test::TestRequest::get() + .uri(&format!("/indexes/{}/documents?", urlencode(index_uid))) + .insert_header((ACCEPT_ENCODING, "gzip")) + .to_request(); + + let res = test::call_service(&app, req).await; + + assert_eq!(res.status(), 200); + + let bytes = test::read_body(res).await; + let decoded = Encoder::Gzip.decode(bytes); + let parsed_response = + serde_json::from_slice::(decoded.into().as_ref()).expect("Expecting valid json"); + + let arr = parsed_response["results"].as_array().unwrap(); + assert_eq!(arr.len(), 20); +} + #[actix_rt::test] async fn test_get_all_documents_limit() { let server = Server::new().await; diff --git a/meilisearch-http/tests/documents/mod.rs b/meilisearch-http/tests/documents/mod.rs index a791a596f..794b57c3a 100644 --- a/meilisearch-http/tests/documents/mod.rs +++ b/meilisearch-http/tests/documents/mod.rs @@ -1,3 +1,4 @@ mod add_documents; mod delete_documents; mod get_documents; +mod update_documents; diff --git a/meilisearch-http/tests/documents/update_documents.rs b/meilisearch-http/tests/documents/update_documents.rs new file mode 100644 index 000000000..99d700f9f --- /dev/null +++ b/meilisearch-http/tests/documents/update_documents.rs @@ -0,0 +1,207 @@ +use crate::common::{GetAllDocumentsOptions, Server}; + +use crate::common::encoder::Encoder; +use serde_json::json; + +#[actix_rt::test] +async fn error_document_update_create_index_bad_uid() { + let server = Server::new().await; + let index = server.index("883 fj!"); + let (response, code) = index.update_documents(json!([{"id": 1}]), None).await; + + let expected_response = json!({ + "message": "invalid index uid `883 fj!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.", + "code": "invalid_index_uid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_index_uid" + }); + + assert_eq!(code, 400); + assert_eq!(response, expected_response); +} + +#[actix_rt::test] +async fn document_update_with_primary_key() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "primary": 1, + "content": "foo", + } + ]); + let (_response, code) = index.update_documents(documents, Some("primary")).await; + assert_eq!(code, 202); + + index.wait_task(0).await; + + let (response, code) = index.get_task(0).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["uid"], 0); + assert_eq!(response["type"], "documentAdditionOrUpdate"); + assert_eq!(response["details"]["indexedDocuments"], 1); + assert_eq!(response["details"]["receivedDocuments"], 1); + + let (response, code) = index.get().await; + assert_eq!(code, 200); + assert_eq!(response["primaryKey"], "primary"); +} + +#[actix_rt::test] +async fn update_document() { + let server = Server::new().await; + let index = server.index("test"); + + let documents = json!([ + { + "doc_id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + + index.wait_task(0).await; + + let documents = json!([ + { + "doc_id": 1, + "other": "bar", + } + ]); + + let (response, code) = index.update_documents(documents, None).await; + assert_eq!(code, 202, "response: {}", response); + + index.wait_task(1).await; + + let (response, code) = index.get_task(1).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "succeeded"); + + let (response, code) = index.get_document(1, None).await; + assert_eq!(code, 200); + assert_eq!( + response.to_string(), + r##"{"doc_id":1,"content":"foo","other":"bar"}"## + ); +} + +#[actix_rt::test] +async fn update_document_gzip_encoded() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Gzip); + + let documents = json!([ + { + "doc_id": 1, + "content": "foo", + } + ]); + + let (_response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + + index.wait_task(0).await; + + let documents = json!([ + { + "doc_id": 1, + "other": "bar", + } + ]); + + let (response, code) = index.update_documents(documents, None).await; + assert_eq!(code, 202, "response: {}", response); + + index.wait_task(1).await; + + let (response, code) = index.get_task(1).await; + assert_eq!(code, 200); + assert_eq!(response["status"], "succeeded"); + + let (response, code) = index.get_document(1, None).await; + assert_eq!(code, 200); + assert_eq!( + response.to_string(), + r##"{"doc_id":1,"content":"foo","other":"bar"}"## + ); +} + +#[actix_rt::test] +async fn update_larger_dataset() { + let server = Server::new().await; + let index = server.index("test"); + let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(); + index.update_documents(documents, None).await; + index.wait_task(0).await; + let (response, code) = index.get_task(0).await; + assert_eq!(code, 200); + assert_eq!(response["type"], "documentAdditionOrUpdate"); + assert_eq!(response["details"]["indexedDocuments"], 77); + let (response, code) = index + .get_all_documents(GetAllDocumentsOptions { + limit: Some(1000), + ..Default::default() + }) + .await; + assert_eq!(code, 200); + assert_eq!(response["results"].as_array().unwrap().len(), 77); +} + +#[actix_rt::test] +async fn error_update_documents_bad_document_id() { + let server = Server::new().await; + let index = server.index("test"); + index.create(Some("docid")).await; + let documents = json!([ + { + "docid": "foo & bar", + "content": "foobar" + } + ]); + index.update_documents(documents, None).await; + let response = index.wait_task(1).await; + assert_eq!(response["status"], json!("failed")); + assert_eq!( + response["error"]["message"], + json!( + r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."# + ) + ); + assert_eq!(response["error"]["code"], json!("invalid_document_id")); + assert_eq!(response["error"]["type"], json!("invalid_request")); + assert_eq!( + response["error"]["link"], + json!("https://docs.meilisearch.com/errors#invalid_document_id") + ); +} + +#[actix_rt::test] +async fn error_update_documents_missing_document_id() { + let server = Server::new().await; + let index = server.index("test"); + index.create(Some("docid")).await; + let documents = json!([ + { + "id": "11", + "content": "foobar" + } + ]); + index.update_documents(documents, None).await; + let response = index.wait_task(1).await; + assert_eq!(response["status"], "failed"); + assert_eq!( + response["error"]["message"], + r#"Document doesn't have a `docid` attribute: `{"id":"11","content":"foobar"}`."# + ); + assert_eq!(response["error"]["code"], "missing_document_id"); + assert_eq!(response["error"]["type"], "invalid_request"); + assert_eq!( + response["error"]["link"], + "https://docs.meilisearch.com/errors#missing_document_id" + ); +} diff --git a/meilisearch-http/tests/index/create_index.rs b/meilisearch-http/tests/index/create_index.rs index a1c508e1f..30040abfe 100644 --- a/meilisearch-http/tests/index/create_index.rs +++ b/meilisearch-http/tests/index/create_index.rs @@ -1,4 +1,9 @@ +use crate::common::encoder::Encoder; use crate::common::Server; +use actix_web::http::header::ContentType; +use actix_web::test; +use http::header::ACCEPT_ENCODING; +use meilisearch_http::{analytics, create_app}; use serde_json::{json, Value}; #[actix_rt::test] @@ -18,6 +23,95 @@ async fn create_index_no_primary_key() { assert_eq!(response["details"]["primaryKey"], Value::Null); } +#[actix_rt::test] +async fn create_index_with_gzip_encoded_request() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Gzip); + let (response, code) = index.create(None).await; + + assert_eq!(code, 202); + + assert_eq!(response["status"], "enqueued"); + + let response = index.wait_task(0).await; + + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexCreation"); + assert_eq!(response["details"]["primaryKey"], Value::Null); +} + +#[actix_rt::test] +async fn create_index_with_gzip_encoded_request_and_receiving_brotli_encoded_response() { + let server = Server::new().await; + let app = test::init_service(create_app!( + &server.service.meilisearch, + &server.service.auth, + true, + server.service.options, + analytics::MockAnalytics::new(&server.service.options).0 + )) + .await; + + let body = serde_json::to_string(&json!({ + "uid": "test", + "primaryKey": None::<&str>, + })) + .unwrap(); + let req = test::TestRequest::post() + .uri("/indexes") + .insert_header(Encoder::Gzip.header().unwrap()) + .insert_header((ACCEPT_ENCODING, "br")) + .insert_header(ContentType::json()) + .set_payload(Encoder::Gzip.encode(body)) + .to_request(); + + let res = test::call_service(&app, req).await; + + assert_eq!(res.status(), 202); + + let bytes = test::read_body(res).await; + let decoded = Encoder::Brotli.decode(bytes); + let parsed_response = + serde_json::from_slice::(decoded.into().as_ref()).expect("Expecting valid json"); + + assert_eq!(parsed_response["taskUid"], 0); + assert_eq!(parsed_response["indexUid"], "test"); +} + +#[actix_rt::test] +async fn create_index_with_zlib_encoded_request() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Deflate); + let (response, code) = index.create(None).await; + + assert_eq!(code, 202); + + assert_eq!(response["status"], "enqueued"); + + let response = index.wait_task(0).await; + + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexCreation"); + assert_eq!(response["details"]["primaryKey"], Value::Null); +} + +#[actix_rt::test] +async fn create_index_with_brotli_encoded_request() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Brotli); + let (response, code) = index.create(None).await; + + assert_eq!(code, 202); + + assert_eq!(response["status"], "enqueued"); + + let response = index.wait_task(0).await; + + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexCreation"); + assert_eq!(response["details"]["primaryKey"], Value::Null); +} + #[actix_rt::test] async fn create_index_with_primary_key() { let server = Server::new().await; diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index 48fde5608..97eecbf83 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -1,3 +1,4 @@ +use crate::common::encoder::Encoder; use crate::common::Server; use serde_json::json; use time::{format_description::well_known::Rfc3339, OffsetDateTime}; @@ -34,6 +35,22 @@ async fn update_primary_key() { assert_eq!(response.as_object().unwrap().len(), 4); } +#[actix_rt::test] +async fn create_and_update_with_different_encoding() { + let server = Server::new().await; + let index = server.index_with_encoder("test", Encoder::Gzip); + let (_, code) = index.create(None).await; + + assert_eq!(code, 202); + + let index = server.index_with_encoder("test", Encoder::Brotli); + index.update(Some("primary")).await; + + let response = index.wait_task(1).await; + + assert_eq!(response["status"], "succeeded"); +} + #[actix_rt::test] async fn update_nothing() { let server = Server::new().await;