Merge #2332

2332: fix(search): formatted field r=curquiza a=irevoire

fix #2318

Co-authored-by: Irevoire <tamo@meilisearch.com>

meilisearch-http/tests/search/formatted.rs

@@ -0,0 +1,376 @@
+use super::*;
+use crate::common::Server;
+use serde_json::json;
+
+#[actix_rt::test]
+async fn formatted_contain_wildcard() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({ "displayedAttributes": ["id", "cattos"] }))
+        .await;
+
+    let documents = NESTED_DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_task(1).await;
+
+    let (response, code) = index
+        .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["father", "mother"], "attributesToHighlight": ["father", "mother", "*"], "attributesToCrop": ["doggos"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "_formatted": {
+                "id": "852",
+                "cattos": "<em>pesti</em>",
+            }
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["*"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+            "cattos": "pesti",
+        })
+    );
+
+    let (response, code) = index
+        .search_post(
+            json!({ "q": "pesti", "attributesToRetrieve": ["*"], "attributesToHighlight": ["id"] }),
+        )
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+            "cattos": "pesti",
+            "_formatted": {
+                "id": "852",
+                "cattos": "pesti",
+            }
+        })
+    );
+
+    let (response, code) = index
+        .search_post(
+            json!({ "q": "pesti", "attributesToRetrieve": ["*"], "attributesToCrop": ["*"] }),
+        )
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+            "cattos": "pesti",
+            "_formatted": {
+                "id": "852",
+                "cattos": "pesti",
+            }
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "q": "pesti", "attributesToCrop": ["*"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+            "cattos": "pesti",
+            "_formatted": {
+                "id": "852",
+                "cattos": "pesti",
+            }
+        })
+    );
+}
+
+#[actix_rt::test]
+async fn format_nested() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = NESTED_DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_task(0).await;
+
+    let (response, code) = index
+        .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["doggos"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "doggos": [
+                {
+                    "name": "bobby",
+                    "age": 2,
+                },
+                {
+                    "name": "buddy",
+                    "age": 4,
+                },
+            ],
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "doggos": [
+                {
+                    "name": "bobby",
+                },
+                {
+                    "name": "buddy",
+                },
+            ],
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.name"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "_formatted": {
+                "doggos": [
+                    {
+                        "name": "bobby",
+                    },
+                    {
+                        "name": "buddy",
+                    },
+                ],
+            },
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToCrop": ["doggos.name"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "_formatted": {
+                "doggos": [
+                    {
+                        "name": "bobby",
+                    },
+                    {
+                        "name": "buddy",
+                    },
+                ],
+            },
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "q": "pesti", "attributesToRetrieve": ["doggos.name"], "attributesToHighlight": ["doggos.age"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "doggos": [
+                {
+                    "name": "bobby",
+                },
+                {
+                    "name": "buddy",
+                },
+            ],
+            "_formatted": {
+                "doggos": [
+                    {
+                        "name": "bobby",
+                        "age": "2",
+                    },
+                    {
+                        "name": "buddy",
+                        "age": "4",
+                    },
+                ],
+            },
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "q": "pesti", "attributesToRetrieve": [], "attributesToHighlight": ["doggos.age"], "attributesToCrop": ["doggos.name"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "_formatted": {
+                "doggos": [
+                    {
+                        "name": "bobby",
+                        "age": "2",
+                    },
+                    {
+                        "name": "buddy",
+                        "age": "4",
+                    },
+                ],
+            },
+        })
+    );
+}
+
+#[actix_rt::test]
+async fn displayedattr_2_smol() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    // not enough displayed for the other settings
+    index
+        .update_settings(json!({ "displayedAttributes": ["id"] }))
+        .await;
+
+    let documents = NESTED_DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_task(1).await;
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToRetrieve": ["father", "id"], "attributesToHighlight": ["mother"], "attributesToCrop": ["cattos"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToRetrieve": ["id"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToHighlight": ["id"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+            "_formatted": {
+                "id": "852",
+            }
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToCrop": ["id"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+            "_formatted": {
+                "id": "852",
+            }
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToHighlight": ["id"], "attributesToCrop": ["id"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+            "_formatted": {
+                "id": "852",
+            }
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToHighlight": ["cattos"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToCrop": ["cattos"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "id": 852,
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToRetrieve": ["cattos"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(response["hits"][0], json!({}));
+
+    let (response, code) = index
+        .search_post(
+            json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["cattos"], "attributesToCrop": ["cattos"] }),
+        )
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(response["hits"][0], json!({}));
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToRetrieve": ["cattos"], "attributesToHighlight": ["id"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "_formatted": {
+                "id": "852",
+            }
+        })
+    );
+
+    let (response, code) = index
+        .search_post(json!({ "attributesToRetrieve": ["cattos"], "attributesToCrop": ["id"] }))
+        .await;
+    assert_eq!(code, 200, "{}", response);
+    assert_eq!(
+        response["hits"][0],
+        json!({
+            "_formatted": {
+                "id": "852",
+            }
+        })
+    );
+}

meilisearch-http/tests/search/mod.rs

@@ -2,12 +2,13 @@
 // should be tested in its own module to isolate tests and keep the tests readable.
 
 mod errors;
+mod formatted;
 
 use crate::common::Server;
 use once_cell::sync::Lazy;
 use serde_json::{json, Value};
 
-static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([
         {
             "title": "Shazam!",
@@ -32,7 +33,7 @@ static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     ])
 });
 
-static NESTED_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+pub(self) static NESTED_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([
         {
             "id": 852,

meilisearch-lib/src/index/search.rs

@@ -232,14 +232,22 @@ impl Index {
         let documents_iter = self.documents(&rtxn, documents_ids)?;
 
         for (_id, obkv) in documents_iter {
-            let mut document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?;
+            // First generate a document with all the displayed fields
+            let displayed_document = make_document(&displayed_ids, &fields_ids_map, obkv)?;
+
+            // select the attributes to retrieve
+            let attributes_to_retrieve = to_retrieve_ids
+                .iter()
+                .map(|&fid| fields_ids_map.name(fid).expect("Missing field name"));
+            let mut document =
+                permissive_json_pointer::select_values(&displayed_document, attributes_to_retrieve);
 
             let matches_info = query
                 .matches
                 .then(|| compute_matches(&matching_words, &document, &analyzer));
 
             let formatted = format_fields(
-                &document,
+                &displayed_document,
                 &fields_ids_map,
                 &formatter,
                 &matching_words,
@@ -475,7 +483,7 @@ fn add_non_formatted_ids_to_formatted_options(
 }
 
 fn make_document(
-    attributes_to_retrieve: &BTreeSet<FieldId>,
+    displayed_attributes: &BTreeSet<FieldId>,
     field_ids_map: &FieldsIdsMap,
     obkv: obkv::KvReaderU16,
 ) -> Result<Document> {
@@ -493,11 +501,11 @@ fn make_document(
     }
 
     // select the attributes to retrieve
-    let attributes_to_retrieve = attributes_to_retrieve
+    let displayed_attributes = displayed_attributes
         .iter()
         .map(|&fid| field_ids_map.name(fid).expect("Missing field name"));
 
-    let document = permissive_json_pointer::select_values(&document, attributes_to_retrieve);
+    let document = permissive_json_pointer::select_values(&document, displayed_attributes);
     Ok(document)
 }
 
@@ -514,7 +522,6 @@ fn format_fields<A: AsRef<[u8]>>(
         // before.
         .map(|&fid| field_ids_map.name(fid).unwrap())
         .collect();
-
     let mut document = permissive_json_pointer::select_values(document, selectors.iter().copied());
 
     permissive_json_pointer::map_leaf_values(&mut document, selectors, |key, value| {