Added support for specifying compression in tests

Refactored tests code to allow to specify compression (content-encoding) algorithm.

Added tests to verify what actix actually handle different content encodings properly.

Cargo.lock

@@ -2038,6 +2038,7 @@ name = "meilisearch-http"
 version = "0.29.1"
 dependencies = [
  "actix-cors",
+ "actix-http",
  "actix-rt",
  "actix-web",
  "actix-web-static-files",
@@ -2045,6 +2046,7 @@ dependencies = [
  "assert-json-diff",
  "async-stream",
  "async-trait",
+ "brotli",
  "bstr 1.0.1",
  "byte-unit",
  "bytes",

meilisearch-http/Cargo.toml

@@ -24,6 +24,7 @@ zip = { version = "0.6.2", optional = true }
 [dependencies]
 actix-cors = "0.6.3"
 actix-web = { version = "4.2.1", default-features = false, features = ["macros", "compress-brotli", "compress-gzip", "cookies", "rustls"] }
+actix-http = { version = "3.2.2" }
 actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true }
 anyhow = { version = "1.0.65", features = ["backtrace"] }
 async-stream = "0.3.3"
@@ -89,6 +90,7 @@ manifest-dir-macros = "0.1.16"
 maplit = "1.0.2"
 urlencoding = "2.1.2"
 yaup = "0.2.1"
+brotli = "3.3.4"
 
 [features]
 default = ["analytics", "meilisearch-lib/default", "mini-dashboard"]

meilisearch-http/tests/common/encoder.rs

@@ -0,0 +1,50 @@
+use std::io::Write;
+use actix_http::header::TryIntoHeaderPair;
+use bytes::Bytes;
+use flate2::write::{GzEncoder, ZlibEncoder};
+use flate2::Compression;
+
+#[derive(Clone,Copy)]
+pub enum Encoder {
+    Plain,
+    Gzip,
+    Deflate,
+    Brotli,
+}
+
+impl Encoder {
+    pub fn encode(self: &Encoder, body: impl Into<Bytes>) -> impl Into<Bytes> {
+        match self {
+            Self::Gzip => {
+                let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+                encoder.write_all(&body.into()).expect("Failed to encode request body");
+                encoder.finish().expect("Failed to encode request body")
+            }
+            Self::Deflate => {
+                let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
+                encoder.write_all(&body.into()).expect("Failed to encode request body");
+                encoder.finish().unwrap()
+            }
+            Self::Plain => Vec::from(body.into()),
+            Self::Brotli => {
+                let mut encoder = brotli::CompressorWriter::new(Vec::new(), 32 * 1024, 3, 22);
+                encoder.write_all(&body.into()).expect("Failed to encode request body");
+                encoder.flush().expect("Failed to encode request body");
+                encoder.into_inner()
+            }
+        }
+    }
+
+    pub fn header(self: &Encoder) -> Option<impl TryIntoHeaderPair> {
+        match self {
+            Self::Plain => None,
+            Self::Gzip => Some(("Content-Encoding", "gzip")),
+            Self::Deflate => Some(("Content-Encoding", "deflate")),
+            Self::Brotli => Some(("Content-Encoding", "br")),
+        }
+    }
+
+    pub fn iterator() -> impl Iterator<Item = Self> {
+        [Self::Plain, Self::Gzip, Self::Deflate, Self::Brotli].iter().copied()
+    }
+}

meilisearch-http/tests/common/index.rs

@@ -7,24 +7,27 @@ use std::{
 use actix_web::http::StatusCode;
 use serde_json::{json, Value};
 use tokio::time::sleep;
-use urlencoding::encode;
+use urlencoding::encode as urlencode;
 
 use super::service::Service;
 
+use super::encoder::Encoder;
+
 pub struct Index<'a> {
     pub uid: String,
     pub service: &'a Service,
+    pub encoder: Encoder,
 }
 
 #[allow(dead_code)]
 impl Index<'_> {
     pub async fn get(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
         self.service.get(url).await
     }
 
     pub async fn load_test_set(&self) -> u64 {
-        let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref()));
         let (response, code) = self
             .service
             .post_str(url, include_str!("../assets/test_set.json"))
@@ -40,20 +43,22 @@ impl Index<'_> {
             "uid": self.uid,
             "primaryKey": primary_key,
         });
-        self.service.post("/indexes", body).await
+        self.service
+            .post_encoded("/indexes", body, self.encoder)
+            .await
     }
 
     pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
         let body = json!({
             "primaryKey": primary_key,
         });
-        let url = format!("/indexes/{}", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
 
-        self.service.patch(url, body).await
+        self.service.patch_encoded(url, body, self.encoder).await
     }
 
     pub async fn delete(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
         self.service.delete(url).await
     }
 
@@ -65,12 +70,14 @@ impl Index<'_> {
         let url = match primary_key {
             Some(key) => format!(
                 "/indexes/{}/documents?primaryKey={}",
-                encode(self.uid.as_ref()),
+                urlencode(self.uid.as_ref()),
                 key
             ),
-            None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
+            None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
         };
-        self.service.post(url, documents).await
+        self.service
+            .post_encoded(url, documents, self.encoder)
+            .await
     }
 
     pub async fn update_documents(
@@ -81,12 +88,12 @@ impl Index<'_> {
         let url = match primary_key {
             Some(key) => format!(
                 "/indexes/{}/documents?primaryKey={}",
-                encode(self.uid.as_ref()),
+                urlencode(self.uid.as_ref()),
                 key
             ),
-            None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
+            None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
         };
-        self.service.put(url, documents).await
+        self.service.put_encoded(url, documents, self.encoder).await
     }
 
     pub async fn wait_task(&self, update_id: u64) -> Value {
@@ -132,7 +139,7 @@ impl Index<'_> {
         id: u64,
         options: Option<GetDocumentOptions>,
     ) -> (Value, StatusCode) {
-        let mut url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
+        let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
         if let Some(fields) = options.and_then(|o| o.fields) {
             let _ = write!(url, "?fields={}", fields.join(","));
         }
@@ -140,7 +147,7 @@ impl Index<'_> {
     }
 
     pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
-        let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref()));
+        let mut url = format!("/indexes/{}/documents?", urlencode(self.uid.as_ref()));
         if let Some(limit) = options.limit {
             let _ = write!(url, "limit={}&", limit);
         }
@@ -157,42 +164,42 @@ impl Index<'_> {
     }
 
     pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
+        let url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
         self.service.delete(url).await
     }
 
     pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref()));
         self.service.delete(url).await
     }
 
     pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
         let url = format!(
             "/indexes/{}/documents/delete-batch",
-            encode(self.uid.as_ref())
+            urlencode(self.uid.as_ref())
         );
         self.service
-            .post(url, serde_json::to_value(&ids).unwrap())
+            .post_encoded(url, serde_json::to_value(&ids).unwrap(), self.encoder)
             .await
     }
 
     pub async fn settings(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
         self.service.get(url).await
     }
 
     pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
-        self.service.patch(url, settings).await
+        self.service.patch_encoded(url, settings, self.encoder).await
     }
 
     pub async fn delete_settings(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
         self.service.delete(url).await
     }
 
     pub async fn stats(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/stats", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/stats", urlencode(self.uid.as_ref()));
         self.service.get(url).await
     }
 
@@ -217,29 +224,29 @@ impl Index<'_> {
     }
 
     pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/search", encode(self.uid.as_ref()));
+        let url = format!("/indexes/{}/search", urlencode(self.uid.as_ref()));
-        self.service.post(url, query).await
+        self.service.post_encoded(url, query, self.encoder).await
     }
 
     pub async fn search_get(&self, query: Value) -> (Value, StatusCode) {
         let params = yaup::to_string(&query).unwrap();
-        let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params);
+        let url = format!("/indexes/{}/search?{}", urlencode(self.uid.as_ref()), params);
         self.service.get(url).await
     }
 
     pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) {
         let url = format!(
             "/indexes/{}/settings/{}",
-            encode(self.uid.as_ref()),
+            urlencode(self.uid.as_ref()),
             "distinct-attribute"
         );
-        self.service.put(url, value).await
+        self.service.put_encoded(url, value, self.encoder).await
     }
 
     pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) {
         let url = format!(
             "/indexes/{}/settings/{}",
-            encode(self.uid.as_ref()),
+            urlencode(self.uid.as_ref()),
             "distinct-attribute"
         );
         self.service.get(url).await

meilisearch-http/tests/common/mod.rs

@@ -1,3 +1,4 @@
+pub mod encoder;
 pub mod index;
 pub mod server;
 pub mod service;

meilisearch-http/tests/common/server.rs

@@ -12,6 +12,7 @@ use once_cell::sync::Lazy;
 use serde_json::Value;
 use tempfile::TempDir;
 
+use crate::common::encoder::Encoder;
 use meilisearch_http::option::Opt;
 
 use super::index::Index;
@@ -100,9 +101,14 @@ impl Server {
 
     /// Returns a view to an index. There is no guarantee that the index exists.
     pub fn index(&self, uid: impl AsRef<str>) -> Index<'_> {
+        self.index_with_encoder(uid, Encoder::Plain)
+    }
+
+    pub fn index_with_encoder(&self, uid: impl AsRef<str>, encoder: Encoder) -> Index<'_> {
         Index {
             uid: uid.as_ref().to_string(),
             service: &self.service,
+            encoder,
         }
     }
 

meilisearch-http/tests/common/service.rs

@@ -1,8 +1,11 @@
+use actix_web::http::header::ContentType;
+use actix_web::test::TestRequest;
 use actix_web::{http::StatusCode, test};
 use meilisearch_auth::AuthController;
 use meilisearch_lib::MeiliSearch;
 use serde_json::Value;
 
+use crate::common::encoder::Encoder;
 use meilisearch_http::{analytics, create_app, Opt};
 
 pub struct Service {
@@ -14,26 +17,18 @@ pub struct Service {
 
 impl Service {
     pub async fn post(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        self.post_encoded(url, body, Encoder::Plain).await
-            &self.meilisearch,
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
-
-        let mut req = test::TestRequest::post().uri(url.as_ref()).set_json(&body);
-        if let Some(api_key) = &self.api_key {
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
     }
-        let req = req.to_request();
-        let res = test::call_service(&app, req).await;
-        let status_code = res.status();
 
-        let body = test::read_body(res).await;
+    pub async fn post_encoded(
-        let response = serde_json::from_slice(&body).unwrap_or_default();
+        &self,
-        (response, status_code)
+        url: impl AsRef<str>,
+        body: Value,
+        encoder: Encoder,
+    ) -> (Value, StatusCode) {
+        let mut req = test::TestRequest::post().uri(url.as_ref());
+        req = self.encode(req, body, encoder);
+        self.request(req).await
     }
 
     /// Send a test post request from a text body, with a `content-type:application/json` header.
@@ -42,101 +37,54 @@ impl Service {
         url: impl AsRef<str>,
         body: impl AsRef<str>,
     ) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        let req = test::TestRequest::post()
-            &self.meilisearch,
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
-
-        let mut req = test::TestRequest::post()
             .uri(url.as_ref())
             .set_payload(body.as_ref().to_string())
             .insert_header(("content-type", "application/json"));
-        if let Some(api_key) = &self.api_key {
+        self.request(req).await
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
-        }
-        let req = req.to_request();
-        let res = test::call_service(&app, req).await;
-        let status_code = res.status();
-
-        let body = test::read_body(res).await;
-        let response = serde_json::from_slice(&body).unwrap_or_default();
-        (response, status_code)
     }
 
     pub async fn get(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        let req = test::TestRequest::get().uri(url.as_ref());
-            &self.meilisearch,
+        self.request(req).await
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
-
-        let mut req = test::TestRequest::get().uri(url.as_ref());
-        if let Some(api_key) = &self.api_key {
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
-        }
-        let req = req.to_request();
-        let res = test::call_service(&app, req).await;
-        let status_code = res.status();
-
-        let body = test::read_body(res).await;
-        let response = serde_json::from_slice(&body).unwrap_or_default();
-        (response, status_code)
     }
 
     pub async fn put(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        self.put_encoded(url, body, Encoder::Plain).await
-            &self.meilisearch,
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
-
-        let mut req = test::TestRequest::put().uri(url.as_ref()).set_json(&body);
-        if let Some(api_key) = &self.api_key {
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
     }
-        let req = req.to_request();
-        let res = test::call_service(&app, req).await;
-        let status_code = res.status();
 
-        let body = test::read_body(res).await;
+    pub async fn put_encoded(
-        let response = serde_json::from_slice(&body).unwrap_or_default();
+        &self,
-        (response, status_code)
+        url: impl AsRef<str>,
+        body: Value,
+        encoder: Encoder,
+    ) -> (Value, StatusCode) {
+        let mut req = test::TestRequest::put().uri(url.as_ref());
+        req = self.encode(req, body, encoder);
+        self.request(req).await
     }
 
     pub async fn patch(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(
+        self.patch_encoded(url, body, Encoder::Plain).await
-            &self.meilisearch,
-            &self.auth,
-            true,
-            self.options,
-            analytics::MockAnalytics::new(&self.options).0
-        ))
-        .await;
-
-        let mut req = test::TestRequest::patch().uri(url.as_ref()).set_json(&body);
-        if let Some(api_key) = &self.api_key {
-            req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
     }
-        let req = req.to_request();
-        let res = test::call_service(&app, req).await;
-        let status_code = res.status();
 
-        let body = test::read_body(res).await;
+    pub async fn patch_encoded(
-        let response = serde_json::from_slice(&body).unwrap_or_default();
+        &self,
-        (response, status_code)
+        url: impl AsRef<str>,
+        body: Value,
+        encoder: Encoder,
+    ) -> (Value, StatusCode) {
+        let mut req = test::TestRequest::patch().uri(url.as_ref());
+        req = self.encode(req, body, encoder);
+        self.request(req).await
     }
 
     pub async fn delete(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
+        let req = test::TestRequest::delete().uri(url.as_ref());
+        self.request(req).await
+    }
+
+    pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) {
         let app = test::init_service(create_app!(
             &self.meilisearch,
             &self.auth,
@@ -146,7 +94,6 @@ impl Service {
         ))
         .await;
 
-        let mut req = test::TestRequest::delete().uri(url.as_ref());
         if let Some(api_key) = &self.api_key {
             req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
         }
@@ -158,4 +105,16 @@ impl Service {
         let response = serde_json::from_slice(&body).unwrap_or_default();
         (response, status_code)
     }
+
+    fn encode(&self, req: TestRequest, body: Value, encoder: Encoder) -> TestRequest {
+        let bytes = serde_json::to_string(&body).expect("Failed to serialize test data to json");
+        let encoded_body = encoder.encode(bytes);
+        let header = encoder.header();
+        match header {
+            Some(header) => req.insert_header(header),
+            None => req,
+        }
+        .set_payload(encoded_body)
+        .insert_header(ContentType::json())
+    }
 }

meilisearch-http/tests/index/create_index.rs

@@ -1,3 +1,4 @@
+use crate::common::encoder::Encoder;
 use crate::common::Server;
 use serde_json::{json, Value};
 
@@ -18,6 +19,57 @@ async fn create_index_no_primary_key() {
     assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
+#[actix_rt::test]
+async fn create_index_with_gzip_encoded_request() {
+    let server = Server::new().await;
+    let index = server.index_with_encoder("test", Encoder::Gzip);
+    let (response, code) = index.create(None).await;
+
+    assert_eq!(code, 202);
+
+    assert_eq!(response["status"], "enqueued");
+
+    let response = index.wait_task(0).await;
+
+    assert_eq!(response["status"], "succeeded");
+    assert_eq!(response["type"], "indexCreation");
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
+}
+
+#[actix_rt::test]
+async fn create_index_with_zlib_encoded_request() {
+    let server = Server::new().await;
+    let index = server.index_with_encoder("test", Encoder::Deflate);
+    let (response, code) = index.create(None).await;
+
+    assert_eq!(code, 202);
+
+    assert_eq!(response["status"], "enqueued");
+
+    let response = index.wait_task(0).await;
+
+    assert_eq!(response["status"], "succeeded");
+    assert_eq!(response["type"], "indexCreation");
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
+}
+
+#[actix_rt::test]
+async fn create_index_with_brotli_encoded_request() {
+    let server = Server::new().await;
+    let index = server.index_with_encoder("test", Encoder::Brotli);
+    let (response, code) = index.create(None).await;
+
+    assert_eq!(code, 202);
+
+    assert_eq!(response["status"], "enqueued");
+
+    let response = index.wait_task(0).await;
+
+    assert_eq!(response["status"], "succeeded");
+    assert_eq!(response["type"], "indexCreation");
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
+}
+
 #[actix_rt::test]
 async fn create_index_with_primary_key() {
     let server = Server::new().await;

meilisearch-http/tests/index/update_index.rs

@@ -1,3 +1,4 @@
+use crate::common::encoder::Encoder;
 use crate::common::Server;
 use serde_json::json;
 use time::{format_description::well_known::Rfc3339, OffsetDateTime};
@@ -34,6 +35,22 @@ async fn update_primary_key() {
     assert_eq!(response.as_object().unwrap().len(), 4);
 }
 
+#[actix_rt::test]
+async fn create_and_update_with_different_encoding() {
+    let server = Server::new().await;
+    let index = server.index_with_encoder("test", Encoder::Gzip);
+    let (_, code) = index.create(None).await;
+
+    assert_eq!(code, 202);
+
+    let index = server.index_with_encoder("test", Encoder::Brotli);
+    index.update(Some("primary")).await;
+
+    let response = index.wait_task(1).await;
+
+    assert_eq!(response["status"], "succeeded");
+}
+
 #[actix_rt::test]
 async fn update_nothing() {
     let server = Server::new().await;