snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 02:27:40 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Merge #2876

Browse Source 
2876: Full support for compressed (Gzip, Brotli, Zlib) API requests r=Kerollmops a=mou

# Pull Request

## Related issue
Fixes #2802 

## What does this PR do?
- Adds missed content-encoding support for streamed requests (documents)
- Adds additional tests to validate content-encoding support is in place
- Adds new tests to validate content-encoding support for previously existing code (built-in actix functionality for unmarshaling JSON payloads)

## PR checklist
Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?

Thank you so much for contributing to Meilisearch!

Co-authored-by: Andrey "MOU" Larionov <anlarionov@gmail.com>
This commit is contained in:
bors[bot] 2022-10-13 15:32:21 +00:00 committed by GitHub
commit fc5a7e376c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 650 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -2038,6 +2038,7 @@ name = "meilisearch-http"
version = "0.29.1" version = "0.29.1"
dependencies = [ dependencies = [
"actix-cors", "actix-cors",
"actix-http",
"actix-rt", "actix-rt",
"actix-web", "actix-web",
"actix-web-static-files", "actix-web-static-files",
@ -2045,6 +2046,7 @@ dependencies = [
"assert-json-diff", "assert-json-diff",
"async-stream", "async-stream",
"async-trait", "async-trait",
"brotli",
"bstr 1.0.1", "bstr 1.0.1",
"byte-unit", "byte-unit",
"bytes", "bytes",

2
meilisearch-http/Cargo.toml
View File

@ -24,6 +24,7 @@ zip = { version = "0.6.2", optional = true }
[dependencies] [dependencies]
actix-cors = "0.6.3" actix-cors = "0.6.3"
actix-web = { version = "4.2.1", default-features = false, features = ["macros", "compress-brotli", "compress-gzip", "cookies", "rustls"] } actix-web = { version = "4.2.1", default-features = false, features = ["macros", "compress-brotli", "compress-gzip", "cookies", "rustls"] }
actix-http = "3.2.2"
actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true } actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true }
anyhow = { version = "1.0.65", features = ["backtrace"] } anyhow = { version = "1.0.65", features = ["backtrace"] }
async-stream = "0.3.3" async-stream = "0.3.3"
@ -85,6 +86,7 @@ lazy_static = "1.4.0"
[dev-dependencies] [dev-dependencies]
actix-rt = "2.7.0" actix-rt = "2.7.0"
assert-json-diff = "2.0.2" assert-json-diff = "2.0.2"
brotli = "3.3.4"
manifest-dir-macros = "0.1.16" manifest-dir-macros = "0.1.16"
maplit = "1.0.2" maplit = "1.0.2"
urlencoding = "2.1.2" urlencoding = "2.1.2"

5
meilisearch-http/src/extractors/payload.rs
View File

@ -1,13 +1,14 @@
use std::pin::Pin; use std::pin::Pin;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use actix_http::encoding::Decoder as Decompress;
use actix_web::error::PayloadError; use actix_web::error::PayloadError;
use actix_web::{dev, web, FromRequest, HttpRequest}; use actix_web::{dev, web, FromRequest, HttpRequest};
use futures::future::{ready, Ready}; use futures::future::{ready, Ready};
use futures::Stream; use futures::Stream;
pub struct Payload { pub struct Payload {
payload: dev::Payload, payload: Decompress<dev::Payload>,
limit: usize, limit: usize,
} }
@ -39,7 +40,7 @@ impl FromRequest for Payload {
.map(|c| c.limit) .map(|c| c.limit)
.unwrap_or(PayloadConfig::default().limit); .unwrap_or(PayloadConfig::default().limit);
ready(Ok(Payload { ready(Ok(Payload {
payload: payload.take(), payload: Decompress::from_headers(payload.take(), req.headers()),
limit, limit,
})) }))
} }

87
meilisearch-http/tests/common/encoder.rs Normal file
View File

@ -0,0 +1,87 @@
use actix_http::header::TryIntoHeaderPair;
use bytes::Bytes;
use flate2::read::{GzDecoder, ZlibDecoder};
use flate2::write::{GzEncoder, ZlibEncoder};
use flate2::Compression;
use std::io::{Read, Write};
#[derive(Clone, Copy)]
pub enum Encoder {
Plain,
Gzip,
Deflate,
Brotli,
}
impl Encoder {
pub fn encode(self: &Encoder, body: impl Into<Bytes>) -> impl Into<Bytes> {
match self {
Self::Gzip => {
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder
.write_all(&body.into())
.expect("Failed to encode request body");
encoder.finish().expect("Failed to encode request body")
}
Self::Deflate => {
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
encoder
.write_all(&body.into())
.expect("Failed to encode request body");
encoder.finish().unwrap()
}
Self::Plain => Vec::from(body.into()),
Self::Brotli => {
let mut encoder = brotli::CompressorWriter::new(Vec::new(), 32 * 1024, 3, 22);
encoder
.write_all(&body.into())
.expect("Failed to encode request body");
encoder.flush().expect("Failed to encode request body");
encoder.into_inner()
}
}
}
pub fn decode(self: &Encoder, bytes: impl Into<Bytes>) -> impl Into<Bytes> {
let mut buffer = Vec::new();
let input = bytes.into();
match self {
Self::Gzip => {
GzDecoder::new(input.as_ref())
.read_to_end(&mut buffer)
.expect("Invalid gzip stream");
}
Self::Deflate => {
ZlibDecoder::new(input.as_ref())
.read_to_end(&mut buffer)
.expect("Invalid zlib stream");
}
Self::Plain => {
buffer
.write_all(input.as_ref())
.expect("Unexpected memory copying issue");
}
Self::Brotli => {
brotli::Decompressor::new(input.as_ref(), 4096)
.read_to_end(&mut buffer)
.expect("Invalid brotli stream");
}
};
buffer
}
pub fn header(self: &Encoder) -> Option<impl TryIntoHeaderPair> {
match self {
Self::Plain => None,
Self::Gzip => Some(("Content-Encoding", "gzip")),
Self::Deflate => Some(("Content-Encoding", "deflate")),
Self::Brotli => Some(("Content-Encoding", "br")),
}
}
pub fn iterator() -> impl Iterator<Item = Self> {
[Self::Plain, Self::Gzip, Self::Deflate, Self::Brotli]
.iter()
.copied()
}
}

73
meilisearch-http/tests/common/index.rs
View File

@ -7,24 +7,27 @@ use std::{
use actix_web::http::StatusCode; use actix_web::http::StatusCode;
use serde_json::{json, Value}; use serde_json::{json, Value};
use tokio::time::sleep; use tokio::time::sleep;
use urlencoding::encode; use urlencoding::encode as urlencode;
use super::service::Service; use super::service::Service;
use super::encoder::Encoder;
pub struct Index<'a> { pub struct Index<'a> {
pub uid: String, pub uid: String,
pub service: &'a Service, pub service: &'a Service,
pub encoder: Encoder,
} }
#[allow(dead_code)] #[allow(dead_code)]
impl Index<'_> { impl Index<'_> {
pub async fn get(&self) -> (Value, StatusCode) { pub async fn get(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}", encode(self.uid.as_ref())); let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
self.service.get(url).await self.service.get(url).await
} }
pub async fn load_test_set(&self) -> u64 { pub async fn load_test_set(&self) -> u64 {
let url = format!("/indexes/{}/documents", encode(self.uid.as_ref())); let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref()));
let (response, code) = self let (response, code) = self
.service .service
.post_str(url, include_str!("../assets/test_set.json")) .post_str(url, include_str!("../assets/test_set.json"))
@ -40,20 +43,22 @@ impl Index<'_> {
"uid": self.uid, "uid": self.uid,
"primaryKey": primary_key, "primaryKey": primary_key,
}); });
self.service.post("/indexes", body).await self.service
.post_encoded("/indexes", body, self.encoder)
.await
} }
pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) { pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
let body = json!({ let body = json!({
"primaryKey": primary_key, "primaryKey": primary_key,
}); });
let url = format!("/indexes/{}", encode(self.uid.as_ref())); let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
self.service.patch(url, body).await self.service.patch_encoded(url, body, self.encoder).await
} }
pub async fn delete(&self) -> (Value, StatusCode) { pub async fn delete(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}", encode(self.uid.as_ref())); let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
self.service.delete(url).await self.service.delete(url).await
} }
@ -65,12 +70,14 @@ impl Index<'_> {
let url = match primary_key { let url = match primary_key {
Some(key) => format!( Some(key) => format!(
"/indexes/{}/documents?primaryKey={}", "/indexes/{}/documents?primaryKey={}",
encode(self.uid.as_ref()), urlencode(self.uid.as_ref()),
key key
), ),
None => format!("/indexes/{}/documents", encode(self.uid.as_ref())), None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
}; };
self.service.post(url, documents).await self.service
.post_encoded(url, documents, self.encoder)
.await
} }
pub async fn update_documents( pub async fn update_documents(
@ -81,12 +88,12 @@ impl Index<'_> {
let url = match primary_key { let url = match primary_key {
Some(key) => format!( Some(key) => format!(
"/indexes/{}/documents?primaryKey={}", "/indexes/{}/documents?primaryKey={}",
encode(self.uid.as_ref()), urlencode(self.uid.as_ref()),
key key
), ),
None => format!("/indexes/{}/documents", encode(self.uid.as_ref())), None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
}; };
self.service.put(url, documents).await self.service.put_encoded(url, documents, self.encoder).await
} }
pub async fn wait_task(&self, update_id: u64) -> Value { pub async fn wait_task(&self, update_id: u64) -> Value {
@ -132,7 +139,7 @@ impl Index<'_> {
id: u64, id: u64,
options: Option<GetDocumentOptions>, options: Option<GetDocumentOptions>,
) -> (Value, StatusCode) { ) -> (Value, StatusCode) {
let mut url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
if let Some(fields) = options.and_then(|o| o.fields) { if let Some(fields) = options.and_then(|o| o.fields) {
let _ = write!(url, "?fields={}", fields.join(",")); let _ = write!(url, "?fields={}", fields.join(","));
} }
@ -140,7 +147,7 @@ impl Index<'_> {
} }
pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) { pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref())); let mut url = format!("/indexes/{}/documents?", urlencode(self.uid.as_ref()));
if let Some(limit) = options.limit { if let Some(limit) = options.limit {
let _ = write!(url, "limit={}&", limit); let _ = write!(url, "limit={}&", limit);
} }
@ -157,42 +164,44 @@ impl Index<'_> {
} }
pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) { pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id); let url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
self.service.delete(url).await self.service.delete(url).await
} }
pub async fn clear_all_documents(&self) -> (Value, StatusCode) { pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents", encode(self.uid.as_ref())); let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref()));
self.service.delete(url).await self.service.delete(url).await
} }
pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) { pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
let url = format!( let url = format!(
"/indexes/{}/documents/delete-batch", "/indexes/{}/documents/delete-batch",
encode(self.uid.as_ref()) urlencode(self.uid.as_ref())
); );
self.service self.service
.post(url, serde_json::to_value(&ids).unwrap()) .post_encoded(url, serde_json::to_value(&ids).unwrap(), self.encoder)
.await .await
} }
pub async fn settings(&self) -> (Value, StatusCode) { pub async fn settings(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
self.service.get(url).await self.service.get(url).await
} }
pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) { pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
self.service.patch(url, settings).await self.service
.patch_encoded(url, settings, self.encoder)
.await
} }
pub async fn delete_settings(&self) -> (Value, StatusCode) { pub async fn delete_settings(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings", encode(self.uid.as_ref())); let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
self.service.delete(url).await self.service.delete(url).await
} }
pub async fn stats(&self) -> (Value, StatusCode) { pub async fn stats(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/stats", encode(self.uid.as_ref())); let url = format!("/indexes/{}/stats", urlencode(self.uid.as_ref()));
self.service.get(url).await self.service.get(url).await
} }
@ -217,29 +226,33 @@ impl Index<'_> {
} }
pub async fn search_post(&self, query: Value) -> (Value, StatusCode) { pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/search", encode(self.uid.as_ref())); let url = format!("/indexes/{}/search", urlencode(self.uid.as_ref()));
self.service.post(url, query).await self.service.post_encoded(url, query, self.encoder).await
} }
pub async fn search_get(&self, query: Value) -> (Value, StatusCode) { pub async fn search_get(&self, query: Value) -> (Value, StatusCode) {
let params = yaup::to_string(&query).unwrap(); let params = yaup::to_string(&query).unwrap();
let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params); let url = format!(
"/indexes/{}/search?{}",
urlencode(self.uid.as_ref()),
params
);
self.service.get(url).await self.service.get(url).await
} }
pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) { pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) {
let url = format!( let url = format!(
"/indexes/{}/settings/{}", "/indexes/{}/settings/{}",
encode(self.uid.as_ref()), urlencode(self.uid.as_ref()),
"distinct-attribute" "distinct-attribute"
); );
self.service.put(url, value).await self.service.put_encoded(url, value, self.encoder).await
} }
pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) { pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) {
let url = format!( let url = format!(
"/indexes/{}/settings/{}", "/indexes/{}/settings/{}",
encode(self.uid.as_ref()), urlencode(self.uid.as_ref()),
"distinct-attribute" "distinct-attribute"
); );
self.service.get(url).await self.service.get(url).await

1
meilisearch-http/tests/common/mod.rs
View File

@ -1,3 +1,4 @@
pub mod encoder;
pub mod index; pub mod index;
pub mod server; pub mod server;
pub mod service; pub mod service;

6
meilisearch-http/tests/common/server.rs
View File

@ -12,6 +12,7 @@ use once_cell::sync::Lazy;
use serde_json::Value; use serde_json::Value;
use tempfile::TempDir; use tempfile::TempDir;
use crate::common::encoder::Encoder;
use meilisearch_http::option::Opt; use meilisearch_http::option::Opt;
use super::index::Index; use super::index::Index;
@ -100,9 +101,14 @@ impl Server {
/// Returns a view to an index. There is no guarantee that the index exists. /// Returns a view to an index. There is no guarantee that the index exists.
pub fn index(&self, uid: impl AsRef<str>) -> Index<'_> { pub fn index(&self, uid: impl AsRef<str>) -> Index<'_> {
self.index_with_encoder(uid, Encoder::Plain)
}
pub fn index_with_encoder(&self, uid: impl AsRef<str>, encoder: Encoder) -> Index<'_> {
Index { Index {
uid: uid.as_ref().to_string(), uid: uid.as_ref().to_string(),
service: &self.service, service: &self.service,
encoder,
} }
} }

149
meilisearch-http/tests/common/service.rs
View File

@ -1,8 +1,11 @@
use actix_web::http::header::ContentType;
use actix_web::test::TestRequest;
use actix_web::{http::StatusCode, test}; use actix_web::{http::StatusCode, test};
use meilisearch_auth::AuthController; use meilisearch_auth::AuthController;
use meilisearch_lib::MeiliSearch; use meilisearch_lib::MeiliSearch;
use serde_json::Value; use serde_json::Value;
use crate::common::encoder::Encoder;
use meilisearch_http::{analytics, create_app, Opt}; use meilisearch_http::{analytics, create_app, Opt};
pub struct Service { pub struct Service {
@ -14,26 +17,18 @@ pub struct Service {
impl Service { impl Service {
pub async fn post(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) { pub async fn post(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
let app = test::init_service(create_app!( self.post_encoded(url, body, Encoder::Plain).await
&self.meilisearch,
&self.auth,
true,
self.options,
analytics::MockAnalytics::new(&self.options).0
))
.await;
let mut req = test::TestRequest::post().uri(url.as_ref()).set_json(&body);
if let Some(api_key) = &self.api_key {
req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
} }
let req = req.to_request();
let res = test::call_service(&app, req).await;
let status_code = res.status();
let body = test::read_body(res).await; pub async fn post_encoded(
let response = serde_json::from_slice(&body).unwrap_or_default(); &self,
(response, status_code) url: impl AsRef<str>,
body: Value,
encoder: Encoder,
) -> (Value, StatusCode) {
let mut req = test::TestRequest::post().uri(url.as_ref());
req = self.encode(req, body, encoder);
self.request(req).await
} }
/// Send a test post request from a text body, with a `content-type:application/json` header. /// Send a test post request from a text body, with a `content-type:application/json` header.
@ -42,101 +37,54 @@ impl Service {
url: impl AsRef<str>, url: impl AsRef<str>,
body: impl AsRef<str>, body: impl AsRef<str>,
) -> (Value, StatusCode) { ) -> (Value, StatusCode) {
let app = test::init_service(create_app!( let req = test::TestRequest::post()
&self.meilisearch,
&self.auth,
true,
self.options,
analytics::MockAnalytics::new(&self.options).0
))
.await;
let mut req = test::TestRequest::post()
.uri(url.as_ref()) .uri(url.as_ref())
.set_payload(body.as_ref().to_string()) .set_payload(body.as_ref().to_string())
.insert_header(("content-type", "application/json")); .insert_header(("content-type", "application/json"));
if let Some(api_key) = &self.api_key { self.request(req).await
req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
}
let req = req.to_request();
let res = test::call_service(&app, req).await;
let status_code = res.status();
let body = test::read_body(res).await;
let response = serde_json::from_slice(&body).unwrap_or_default();
(response, status_code)
} }
pub async fn get(&self, url: impl AsRef<str>) -> (Value, StatusCode) { pub async fn get(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
let app = test::init_service(create_app!( let req = test::TestRequest::get().uri(url.as_ref());
&self.meilisearch, self.request(req).await
&self.auth,
true,
self.options,
analytics::MockAnalytics::new(&self.options).0
))
.await;
let mut req = test::TestRequest::get().uri(url.as_ref());
if let Some(api_key) = &self.api_key {
req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
}
let req = req.to_request();
let res = test::call_service(&app, req).await;
let status_code = res.status();
let body = test::read_body(res).await;
let response = serde_json::from_slice(&body).unwrap_or_default();
(response, status_code)
} }
pub async fn put(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) { pub async fn put(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
let app = test::init_service(create_app!( self.put_encoded(url, body, Encoder::Plain).await
&self.meilisearch,
&self.auth,
true,
self.options,
analytics::MockAnalytics::new(&self.options).0
))
.await;
let mut req = test::TestRequest::put().uri(url.as_ref()).set_json(&body);
if let Some(api_key) = &self.api_key {
req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
} }
let req = req.to_request();
let res = test::call_service(&app, req).await;
let status_code = res.status();
let body = test::read_body(res).await; pub async fn put_encoded(
let response = serde_json::from_slice(&body).unwrap_or_default(); &self,
(response, status_code) url: impl AsRef<str>,
body: Value,
encoder: Encoder,
) -> (Value, StatusCode) {
let mut req = test::TestRequest::put().uri(url.as_ref());
req = self.encode(req, body, encoder);
self.request(req).await
} }
pub async fn patch(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) { pub async fn patch(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
let app = test::init_service(create_app!( self.patch_encoded(url, body, Encoder::Plain).await
&self.meilisearch,
&self.auth,
true,
self.options,
analytics::MockAnalytics::new(&self.options).0
))
.await;
let mut req = test::TestRequest::patch().uri(url.as_ref()).set_json(&body);
if let Some(api_key) = &self.api_key {
req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
} }
let req = req.to_request();
let res = test::call_service(&app, req).await;
let status_code = res.status();
let body = test::read_body(res).await; pub async fn patch_encoded(
let response = serde_json::from_slice(&body).unwrap_or_default(); &self,
(response, status_code) url: impl AsRef<str>,
body: Value,
encoder: Encoder,
) -> (Value, StatusCode) {
let mut req = test::TestRequest::patch().uri(url.as_ref());
req = self.encode(req, body, encoder);
self.request(req).await
} }
pub async fn delete(&self, url: impl AsRef<str>) -> (Value, StatusCode) { pub async fn delete(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
let req = test::TestRequest::delete().uri(url.as_ref());
self.request(req).await
}
pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) {
let app = test::init_service(create_app!( let app = test::init_service(create_app!(
&self.meilisearch, &self.meilisearch,
&self.auth, &self.auth,
@ -146,7 +94,6 @@ impl Service {
)) ))
.await; .await;
let mut req = test::TestRequest::delete().uri(url.as_ref());
if let Some(api_key) = &self.api_key { if let Some(api_key) = &self.api_key {
req = req.insert_header(("Authorization", ["Bearer ", api_key].concat())); req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
} }
@ -158,4 +105,16 @@ impl Service {
let response = serde_json::from_slice(&body).unwrap_or_default(); let response = serde_json::from_slice(&body).unwrap_or_default();
(response, status_code) (response, status_code)
} }
fn encode(&self, req: TestRequest, body: Value, encoder: Encoder) -> TestRequest {
let bytes = serde_json::to_string(&body).expect("Failed to serialize test data to json");
let encoded_body = encoder.encode(bytes);
let header = encoder.header();
match header {
Some(header) => req.insert_header(header),
None => req,
}
.set_payload(encoded_body)
.insert_header(ContentType::json())
}
} }

252
meilisearch-http/tests/documents/add_documents.rs
View File

@ -1,6 +1,7 @@
use crate::common::{GetAllDocumentsOptions, Server}; use crate::common::{GetAllDocumentsOptions, Server};
use actix_web::test; use actix_web::test;
use crate::common::encoder::Encoder;
use meilisearch_http::{analytics, create_app}; use meilisearch_http::{analytics, create_app};
use serde_json::{json, Value}; use serde_json::{json, Value};
use time::{format_description::well_known::Rfc3339, OffsetDateTime}; use time::{format_description::well_known::Rfc3339, OffsetDateTime};
@ -97,6 +98,95 @@ async fn add_single_document_test_json_content_types() {
assert_eq!(response["taskUid"], 1); assert_eq!(response["taskUid"], 1);
} }
/// Here we try sending encoded (compressed) document request
#[actix_rt::test]
async fn add_single_document_gzip_encoded() {
let document = json!({
"id": 1,
"content": "Bouvier Bernois",
});
// this is a what is expected and should work
let server = Server::new().await;
let app = test::init_service(create_app!(
&server.service.meilisearch,
&server.service.auth,
true,
server.service.options,
analytics::MockAnalytics::new(&server.service.options).0
))
.await;
// post
let document = serde_json::to_string(&document).unwrap();
let encoder = Encoder::Gzip;
let req = test::TestRequest::post()
.uri("/indexes/dog/documents")
.set_payload(encoder.encode(document.clone()))
.insert_header(("content-type", "application/json"))
.insert_header(encoder.header().unwrap())
.to_request();
let res = test::call_service(&app, req).await;
let status_code = res.status();
let body = test::read_body(res).await;
let response: Value = serde_json::from_slice(&body).unwrap_or_default();
assert_eq!(status_code, 202);
assert_eq!(response["taskUid"], 0);
// put
let req = test::TestRequest::put()
.uri("/indexes/dog/documents")
.set_payload(encoder.encode(document))
.insert_header(("content-type", "application/json"))
.insert_header(encoder.header().unwrap())
.to_request();
let res = test::call_service(&app, req).await;
let status_code = res.status();
let body = test::read_body(res).await;
let response: Value = serde_json::from_slice(&body).unwrap_or_default();
assert_eq!(status_code, 202);
assert_eq!(response["taskUid"], 1);
}
/// Here we try document request with every encoding
#[actix_rt::test]
async fn add_single_document_with_every_encoding() {
let document = json!({
"id": 1,
"content": "Bouvier Bernois",
});
// this is a what is expected and should work
let server = Server::new().await;
let app = test::init_service(create_app!(
&server.service.meilisearch,
&server.service.auth,
true,
server.service.options,
analytics::MockAnalytics::new(&server.service.options).0
))
.await;
// post
let document = serde_json::to_string(&document).unwrap();
for (task_uid, encoder) in Encoder::iterator().enumerate() {
let mut req = test::TestRequest::post()
.uri("/indexes/dog/documents")
.set_payload(encoder.encode(document.clone()))
.insert_header(("content-type", "application/json"));
req = match encoder.header() {
Some(header) => req.insert_header(header),
None => req,
};
let req = req.to_request();
let res = test::call_service(&app, req).await;
let status_code = res.status();
let body = test::read_body(res).await;
let response: Value = serde_json::from_slice(&body).unwrap_or_default();
assert_eq!(status_code, 202);
assert_eq!(response["taskUid"], task_uid);
}
}
/// any other content-type is must be refused /// any other content-type is must be refused
#[actix_rt::test] #[actix_rt::test]
async fn error_add_documents_test_bad_content_types() { async fn error_add_documents_test_bad_content_types() {
@ -690,23 +780,6 @@ async fn error_document_add_create_index_bad_uid() {
assert_eq!(response, expected_response); assert_eq!(response, expected_response);
} }
#[actix_rt::test]
async fn error_document_update_create_index_bad_uid() {
let server = Server::new().await;
let index = server.index("883 fj!");
let (response, code) = index.update_documents(json!([{"id": 1}]), None).await;
let expected_response = json!({
"message": "invalid index uid `883 fj!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.",
"code": "invalid_index_uid",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_index_uid"
});
assert_eq!(code, 400);
assert_eq!(response, expected_response);
}
#[actix_rt::test] #[actix_rt::test]
async fn document_addition_with_primary_key() { async fn document_addition_with_primary_key() {
let server = Server::new().await; let server = Server::new().await;
@ -736,35 +809,6 @@ async fn document_addition_with_primary_key() {
assert_eq!(response["primaryKey"], "primary"); assert_eq!(response["primaryKey"], "primary");
} }
#[actix_rt::test]
async fn document_update_with_primary_key() {
let server = Server::new().await;
let index = server.index("test");
let documents = json!([
{
"primary": 1,
"content": "foo",
}
]);
let (_response, code) = index.update_documents(documents, Some("primary")).await;
assert_eq!(code, 202);
index.wait_task(0).await;
let (response, code) = index.get_task(0).await;
assert_eq!(code, 200);
assert_eq!(response["status"], "succeeded");
assert_eq!(response["uid"], 0);
assert_eq!(response["type"], "documentAdditionOrUpdate");
assert_eq!(response["details"]["indexedDocuments"], 1);
assert_eq!(response["details"]["receivedDocuments"], 1);
let (response, code) = index.get().await;
assert_eq!(code, 200);
assert_eq!(response["primaryKey"], "primary");
}
#[actix_rt::test] #[actix_rt::test]
async fn replace_document() { async fn replace_document() {
let server = Server::new().await; let server = Server::new().await;
@ -811,47 +855,6 @@ async fn add_no_documents() {
assert_eq!(code, 202); assert_eq!(code, 202);
} }
#[actix_rt::test]
async fn update_document() {
let server = Server::new().await;
let index = server.index("test");
let documents = json!([
{
"doc_id": 1,
"content": "foo",
}
]);
let (_response, code) = index.add_documents(documents, None).await;
assert_eq!(code, 202);
index.wait_task(0).await;
let documents = json!([
{
"doc_id": 1,
"other": "bar",
}
]);
let (response, code) = index.update_documents(documents, None).await;
assert_eq!(code, 202, "response: {}", response);
index.wait_task(1).await;
let (response, code) = index.get_task(1).await;
assert_eq!(code, 200);
assert_eq!(response["status"], "succeeded");
let (response, code) = index.get_document(1, None).await;
assert_eq!(code, 200);
assert_eq!(
response.to_string(),
r##"{"doc_id":1,"content":"foo","other":"bar"}"##
);
}
#[actix_rt::test] #[actix_rt::test]
async fn add_larger_dataset() { async fn add_larger_dataset() {
let server = Server::new().await; let server = Server::new().await;
@ -873,27 +876,6 @@ async fn add_larger_dataset() {
assert_eq!(response["results"].as_array().unwrap().len(), 77); assert_eq!(response["results"].as_array().unwrap().len(), 77);
} }
#[actix_rt::test]
async fn update_larger_dataset() {
let server = Server::new().await;
let index = server.index("test");
let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap();
index.update_documents(documents, None).await;
index.wait_task(0).await;
let (response, code) = index.get_task(0).await;
assert_eq!(code, 200);
assert_eq!(response["type"], "documentAdditionOrUpdate");
assert_eq!(response["details"]["indexedDocuments"], 77);
let (response, code) = index
.get_all_documents(GetAllDocumentsOptions {
limit: Some(1000),
..Default::default()
})
.await;
assert_eq!(code, 200);
assert_eq!(response["results"].as_array().unwrap().len(), 77);
}
#[actix_rt::test] #[actix_rt::test]
async fn error_add_documents_bad_document_id() { async fn error_add_documents_bad_document_id() {
let server = Server::new().await; let server = Server::new().await;
@ -924,34 +906,6 @@ async fn error_add_documents_bad_document_id() {
); );
} }
#[actix_rt::test]
async fn error_update_documents_bad_document_id() {
let server = Server::new().await;
let index = server.index("test");
index.create(Some("docid")).await;
let documents = json!([
{
"docid": "foo & bar",
"content": "foobar"
}
]);
index.update_documents(documents, None).await;
let response = index.wait_task(1).await;
assert_eq!(response["status"], json!("failed"));
assert_eq!(
response["error"]["message"],
json!(
r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."#
)
);
assert_eq!(response["error"]["code"], json!("invalid_document_id"));
assert_eq!(response["error"]["type"], json!("invalid_request"));
assert_eq!(
response["error"]["link"],
json!("https://docs.meilisearch.com/errors#invalid_document_id")
);
}
#[actix_rt::test] #[actix_rt::test]
async fn error_add_documents_missing_document_id() { async fn error_add_documents_missing_document_id() {
let server = Server::new().await; let server = Server::new().await;
@ -980,32 +934,6 @@ async fn error_add_documents_missing_document_id() {
); );
} }
#[actix_rt::test]
async fn error_update_documents_missing_document_id() {
let server = Server::new().await;
let index = server.index("test");
index.create(Some("docid")).await;
let documents = json!([
{
"id": "11",
"content": "foobar"
}
]);
index.update_documents(documents, None).await;
let response = index.wait_task(1).await;
assert_eq!(response["status"], "failed");
assert_eq!(
response["error"]["message"],
r#"Document doesn't have a `docid` attribute: `{"id":"11","content":"foobar"}`."#
);
assert_eq!(response["error"]["code"], "missing_document_id");
assert_eq!(response["error"]["type"], "invalid_request");
assert_eq!(
response["error"]["link"],
"https://docs.meilisearch.com/errors#missing_document_id"
);
}
#[actix_rt::test] #[actix_rt::test]
#[ignore] // // TODO: Fix in an other PR: this does not provoke any error. #[ignore] // // TODO: Fix in an other PR: this does not provoke any error.
async fn error_document_field_limit_reached() { async fn error_document_field_limit_reached() {

41
meilisearch-http/tests/documents/get_documents.rs
View File

@ -1,6 +1,11 @@
use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server}; use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server};
use actix_web::test;
use http::header::ACCEPT_ENCODING;
use serde_json::json; use crate::common::encoder::Encoder;
use meilisearch_http::{analytics, create_app};
use serde_json::{json, Value};
use urlencoding::encode as urlencode;
// TODO: partial test since we are testing error, amd error is not yet fully implemented in // TODO: partial test since we are testing error, amd error is not yet fully implemented in
// transplant // transplant
@ -155,6 +160,40 @@ async fn get_all_documents_no_options() {
assert_eq!(first, arr[0]); assert_eq!(first, arr[0]);
} }
#[actix_rt::test]
async fn get_all_documents_no_options_with_response_compression() {
let server = Server::new().await;
let index_uid = "test";
let index = server.index(index_uid);
index.load_test_set().await;
let app = test::init_service(create_app!(
&server.service.meilisearch,
&server.service.auth,
true,
server.service.options,
analytics::MockAnalytics::new(&server.service.options).0
))
.await;
let req = test::TestRequest::get()
.uri(&format!("/indexes/{}/documents?", urlencode(index_uid)))
.insert_header((ACCEPT_ENCODING, "gzip"))
.to_request();
let res = test::call_service(&app, req).await;
assert_eq!(res.status(), 200);
let bytes = test::read_body(res).await;
let decoded = Encoder::Gzip.decode(bytes);
let parsed_response =
serde_json::from_slice::<Value>(decoded.into().as_ref()).expect("Expecting valid json");
let arr = parsed_response["results"].as_array().unwrap();
assert_eq!(arr.len(), 20);
}
#[actix_rt::test] #[actix_rt::test]
async fn test_get_all_documents_limit() { async fn test_get_all_documents_limit() {
let server = Server::new().await; let server = Server::new().await;

1
meilisearch-http/tests/documents/mod.rs
View File

@ -1,3 +1,4 @@
mod add_documents; mod add_documents;
mod delete_documents; mod delete_documents;
mod get_documents; mod get_documents;
mod update_documents;

207
meilisearch-http/tests/documents/update_documents.rs Normal file
View File

@ -0,0 +1,207 @@
use crate::common::{GetAllDocumentsOptions, Server};
use crate::common::encoder::Encoder;
use serde_json::json;
#[actix_rt::test]
async fn error_document_update_create_index_bad_uid() {
let server = Server::new().await;
let index = server.index("883 fj!");
let (response, code) = index.update_documents(json!([{"id": 1}]), None).await;
let expected_response = json!({
"message": "invalid index uid `883 fj!`, the uid must be an integer or a string containing only alphanumeric characters a-z A-Z 0-9, hyphens - and underscores _.",
"code": "invalid_index_uid",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_index_uid"
});
assert_eq!(code, 400);
assert_eq!(response, expected_response);
}
#[actix_rt::test]
async fn document_update_with_primary_key() {
let server = Server::new().await;
let index = server.index("test");
let documents = json!([
{
"primary": 1,
"content": "foo",
}
]);
let (_response, code) = index.update_documents(documents, Some("primary")).await;
assert_eq!(code, 202);
index.wait_task(0).await;
let (response, code) = index.get_task(0).await;
assert_eq!(code, 200);
assert_eq!(response["status"], "succeeded");
assert_eq!(response["uid"], 0);
assert_eq!(response["type"], "documentAdditionOrUpdate");
assert_eq!(response["details"]["indexedDocuments"], 1);
assert_eq!(response["details"]["receivedDocuments"], 1);
let (response, code) = index.get().await;
assert_eq!(code, 200);
assert_eq!(response["primaryKey"], "primary");
}
#[actix_rt::test]
async fn update_document() {
let server = Server::new().await;
let index = server.index("test");
let documents = json!([
{
"doc_id": 1,
"content": "foo",
}
]);
let (_response, code) = index.add_documents(documents, None).await;
assert_eq!(code, 202);
index.wait_task(0).await;
let documents = json!([
{
"doc_id": 1,
"other": "bar",
}
]);
let (response, code) = index.update_documents(documents, None).await;
assert_eq!(code, 202, "response: {}", response);
index.wait_task(1).await;
let (response, code) = index.get_task(1).await;
assert_eq!(code, 200);
assert_eq!(response["status"], "succeeded");
let (response, code) = index.get_document(1, None).await;
assert_eq!(code, 200);
assert_eq!(
response.to_string(),
r##"{"doc_id":1,"content":"foo","other":"bar"}"##
);
}
#[actix_rt::test]
async fn update_document_gzip_encoded() {
let server = Server::new().await;
let index = server.index_with_encoder("test", Encoder::Gzip);
let documents = json!([
{
"doc_id": 1,
"content": "foo",
}
]);
let (_response, code) = index.add_documents(documents, None).await;
assert_eq!(code, 202);
index.wait_task(0).await;
let documents = json!([
{
"doc_id": 1,
"other": "bar",
}
]);
let (response, code) = index.update_documents(documents, None).await;
assert_eq!(code, 202, "response: {}", response);
index.wait_task(1).await;
let (response, code) = index.get_task(1).await;
assert_eq!(code, 200);
assert_eq!(response["status"], "succeeded");
let (response, code) = index.get_document(1, None).await;
assert_eq!(code, 200);
assert_eq!(
response.to_string(),
r##"{"doc_id":1,"content":"foo","other":"bar"}"##
);
}
#[actix_rt::test]
async fn update_larger_dataset() {
let server = Server::new().await;
let index = server.index("test");
let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap();
index.update_documents(documents, None).await;
index.wait_task(0).await;
let (response, code) = index.get_task(0).await;
assert_eq!(code, 200);
assert_eq!(response["type"], "documentAdditionOrUpdate");
assert_eq!(response["details"]["indexedDocuments"], 77);
let (response, code) = index
.get_all_documents(GetAllDocumentsOptions {
limit: Some(1000),
..Default::default()
})
.await;
assert_eq!(code, 200);
assert_eq!(response["results"].as_array().unwrap().len(), 77);
}
#[actix_rt::test]
async fn error_update_documents_bad_document_id() {
let server = Server::new().await;
let index = server.index("test");
index.create(Some("docid")).await;
let documents = json!([
{
"docid": "foo & bar",
"content": "foobar"
}
]);
index.update_documents(documents, None).await;
let response = index.wait_task(1).await;
assert_eq!(response["status"], json!("failed"));
assert_eq!(
response["error"]["message"],
json!(
r#"Document identifier `"foo & bar"` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."#
)
);
assert_eq!(response["error"]["code"], json!("invalid_document_id"));
assert_eq!(response["error"]["type"], json!("invalid_request"));
assert_eq!(
response["error"]["link"],
json!("https://docs.meilisearch.com/errors#invalid_document_id")
);
}
#[actix_rt::test]
async fn error_update_documents_missing_document_id() {
let server = Server::new().await;
let index = server.index("test");
index.create(Some("docid")).await;
let documents = json!([
{
"id": "11",
"content": "foobar"
}
]);
index.update_documents(documents, None).await;
let response = index.wait_task(1).await;
assert_eq!(response["status"], "failed");
assert_eq!(
response["error"]["message"],
r#"Document doesn't have a `docid` attribute: `{"id":"11","content":"foobar"}`."#
);
assert_eq!(response["error"]["code"], "missing_document_id");
assert_eq!(response["error"]["type"], "invalid_request");
assert_eq!(
response["error"]["link"],
"https://docs.meilisearch.com/errors#missing_document_id"
);
}

94
meilisearch-http/tests/index/create_index.rs
View File

@ -1,4 +1,9 @@
use crate::common::encoder::Encoder;
use crate::common::Server; use crate::common::Server;
use actix_web::http::header::ContentType;
use actix_web::test;
use http::header::ACCEPT_ENCODING;
use meilisearch_http::{analytics, create_app};
use serde_json::{json, Value}; use serde_json::{json, Value};
#[actix_rt::test] #[actix_rt::test]
@ -18,6 +23,95 @@ async fn create_index_no_primary_key() {
assert_eq!(response["details"]["primaryKey"], Value::Null); assert_eq!(response["details"]["primaryKey"], Value::Null);
} }
#[actix_rt::test]
async fn create_index_with_gzip_encoded_request() {
let server = Server::new().await;
let index = server.index_with_encoder("test", Encoder::Gzip);
let (response, code) = index.create(None).await;
assert_eq!(code, 202);
assert_eq!(response["status"], "enqueued");
let response = index.wait_task(0).await;
assert_eq!(response["status"], "succeeded");
assert_eq!(response["type"], "indexCreation");
assert_eq!(response["details"]["primaryKey"], Value::Null);
}
#[actix_rt::test]
async fn create_index_with_gzip_encoded_request_and_receiving_brotli_encoded_response() {
let server = Server::new().await;
let app = test::init_service(create_app!(
&server.service.meilisearch,
&server.service.auth,
true,
server.service.options,
analytics::MockAnalytics::new(&server.service.options).0
))
.await;
let body = serde_json::to_string(&json!({
"uid": "test",
"primaryKey": None::<&str>,
}))
.unwrap();
let req = test::TestRequest::post()
.uri("/indexes")
.insert_header(Encoder::Gzip.header().unwrap())
.insert_header((ACCEPT_ENCODING, "br"))
.insert_header(ContentType::json())
.set_payload(Encoder::Gzip.encode(body))
.to_request();
let res = test::call_service(&app, req).await;
assert_eq!(res.status(), 202);
let bytes = test::read_body(res).await;
let decoded = Encoder::Brotli.decode(bytes);
let parsed_response =
serde_json::from_slice::<Value>(decoded.into().as_ref()).expect("Expecting valid json");
assert_eq!(parsed_response["taskUid"], 0);
assert_eq!(parsed_response["indexUid"], "test");
}
#[actix_rt::test]
async fn create_index_with_zlib_encoded_request() {
let server = Server::new().await;
let index = server.index_with_encoder("test", Encoder::Deflate);
let (response, code) = index.create(None).await;
assert_eq!(code, 202);
assert_eq!(response["status"], "enqueued");
let response = index.wait_task(0).await;
assert_eq!(response["status"], "succeeded");
assert_eq!(response["type"], "indexCreation");
assert_eq!(response["details"]["primaryKey"], Value::Null);
}
#[actix_rt::test]
async fn create_index_with_brotli_encoded_request() {
let server = Server::new().await;
let index = server.index_with_encoder("test", Encoder::Brotli);
let (response, code) = index.create(None).await;
assert_eq!(code, 202);
assert_eq!(response["status"], "enqueued");
let response = index.wait_task(0).await;
assert_eq!(response["status"], "succeeded");
assert_eq!(response["type"], "indexCreation");
assert_eq!(response["details"]["primaryKey"], Value::Null);
}
#[actix_rt::test] #[actix_rt::test]
async fn create_index_with_primary_key() { async fn create_index_with_primary_key() {
let server = Server::new().await; let server = Server::new().await;

17
meilisearch-http/tests/index/update_index.rs
View File

@ -1,3 +1,4 @@
use crate::common::encoder::Encoder;
use crate::common::Server; use crate::common::Server;
use serde_json::json; use serde_json::json;
use time::{format_description::well_known::Rfc3339, OffsetDateTime}; use time::{format_description::well_known::Rfc3339, OffsetDateTime};
@ -34,6 +35,22 @@ async fn update_primary_key() {
assert_eq!(response.as_object().unwrap().len(), 4); assert_eq!(response.as_object().unwrap().len(), 4);
} }
#[actix_rt::test]
async fn create_and_update_with_different_encoding() {
let server = Server::new().await;
let index = server.index_with_encoder("test", Encoder::Gzip);
let (_, code) = index.create(None).await;
assert_eq!(code, 202);
let index = server.index_with_encoder("test", Encoder::Brotli);
index.update(Some("primary")).await;
let response = index.wait_task(1).await;
assert_eq!(response["status"], "succeeded");
}
#[actix_rt::test] #[actix_rt::test]
async fn update_nothing() { async fn update_nothing() {
let server = Server::new().await; let server = Server::new().await;