Merge #2427

2427: Update the documents resource r=MarinPostma a=irevoire

- Return Documents API resources on `/documents` in an array in the results field.
- Add limit, offset, and total in the response body.
- Rename `attributesToRetrieve` into `fields` (only for the `/documents` endpoints, not for the `/search` ones).
- The `displayedAttributes` settings do not impact anymore the displayed fields returned in the `/documents` endpoints. These settings only impact the `/search` endpoint.
- make the `/document/:uid` route accept the `fields` query parameter

Fix #2372


Co-authored-by: Irevoire <tamo@meilisearch.com>

meilisearch-http/src/routes/indexes/documents.rs

@@ -13,7 +13,8 @@ use meilisearch_lib::MeiliSearch;
 use mime::Mime;
 use once_cell::sync::Lazy;
 use serde::Deserialize;
-use serde_json::Value;
+use serde_cs::vec::CS;
+use serde_json::{json, Value};
 use tokio::sync::mpsc;
 
 use crate::analytics::Analytics;
@@ -21,11 +22,9 @@ use crate::error::MeilisearchHttpError;
 use crate::extractors::authentication::{policies::*, GuardedData};
 use crate::extractors::payload::Payload;
 use crate::extractors::sequential_extractor::SeqHandler;
+use crate::routes::{fold_star_or, StarOr};
 use crate::task::SummarizedTaskView;
 
-const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0;
-const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20;
-
 static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
     vec![
         "application/json".to_string(),
@@ -86,14 +85,24 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
     );
 }
 
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct GetDocument {
+    fields: Option<CS<StarOr<String>>>,
+}
+
 pub async fn get_document(
     meilisearch: GuardedData<ActionPolicy<{ actions::DOCUMENTS_GET }>, MeiliSearch>,
     path: web::Path<DocumentParam>,
+    params: web::Query<GetDocument>,
 ) -> Result<HttpResponse, ResponseError> {
     let index = path.index_uid.clone();
     let id = path.document_id.clone();
+    let GetDocument { fields } = params.into_inner();
+    let attributes_to_retrieve = fields.map(CS::into_inner).and_then(fold_star_or);
+
     let document = meilisearch
-        .document(index, id, None as Option<Vec<String>>)
+        .document(index, id, attributes_to_retrieve)
         .await?;
     debug!("returns: {:?}", document);
     Ok(HttpResponse::Ok().json(document))
@@ -113,12 +122,16 @@ pub async fn delete_document(
     Ok(HttpResponse::Accepted().json(task))
 }
 
+const PAGINATION_DEFAULT_LIMIT: fn() -> usize = || 20;
+
 #[derive(Deserialize, Debug)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct BrowseQuery {
-    offset: Option<usize>,
+    #[serde(default)]
-    limit: Option<usize>,
+    offset: usize,
-    attributes_to_retrieve: Option<String>,
+    #[serde(default = "PAGINATION_DEFAULT_LIMIT")]
+    limit: usize,
+    fields: Option<CS<StarOr<String>>>,
 }
 
 pub async fn get_all_documents(
@@ -127,27 +140,21 @@ pub async fn get_all_documents(
     params: web::Query<BrowseQuery>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!("called with params: {:?}", params);
-    let attributes_to_retrieve = params.attributes_to_retrieve.as_ref().and_then(|attrs| {
+    let BrowseQuery {
-        let mut names = Vec::new();
+        offset,
-        for name in attrs.split(',').map(String::from) {
+        limit,
-            if name == "*" {
+        fields,
-                return None;
+    } = params.into_inner();
-            }
+    let attributes_to_retrieve = fields.map(CS::into_inner).and_then(fold_star_or);
-            names.push(name);
-        }
-        Some(names)
-    });
 
-    let documents = meilisearch
+    let (total, documents) = meilisearch
-        .documents(
+        .documents(path.into_inner(), offset, limit, attributes_to_retrieve)
-            path.into_inner(),
-            params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET),
-            params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT),
-            attributes_to_retrieve,
-        )
         .await?;
+
     debug!("returns: {:?}", documents);
-    Ok(HttpResponse::Ok().json(documents))
+    Ok(HttpResponse::Ok().json(json!(
+        { "limit": limit, "offset": offset, "total": total, "results": documents }
+    )))
 }
 
 #[derive(Deserialize, Debug)]

meilisearch-http/src/routes/mod.rs

@@ -1,3 +1,5 @@
+use std::str::FromStr;
+
 use actix_web::{web, HttpResponse};
 use log::debug;
 use serde::{Deserialize, Serialize};
@@ -24,6 +26,38 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::scope("/indexes").configure(indexes::configure));
 }
 
+/// A type that tries to match either a star (*) or
+/// any other thing that implements `FromStr`.
+#[derive(Debug)]
+pub enum StarOr<T> {
+    Star,
+    Other(T),
+}
+
+impl<T: FromStr> FromStr for StarOr<T> {
+    type Err = T::Err;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if s.trim() == "*" {
+            Ok(StarOr::Star)
+        } else {
+            T::from_str(s).map(StarOr::Other)
+        }
+    }
+}
+
+/// Extracts the raw values from the `StarOr` types and
+/// return None if a `StarOr::Star` is encountered.
+pub fn fold_star_or<T>(content: impl IntoIterator<Item = StarOr<T>>) -> Option<Vec<T>> {
+    content
+        .into_iter()
+        .map(|value| match value {
+            StarOr::Star => None,
+            StarOr::Other(val) => Some(val),
+        })
+        .collect()
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[allow(clippy::large_enum_variant)]
 #[serde(tag = "name")]

meilisearch-http/src/routes/tasks.rs

@@ -6,13 +6,14 @@ use meilisearch_lib::{IndexUid, MeiliSearch};
 use serde::Deserialize;
 use serde_cs::vec::CS;
 use serde_json::json;
-use std::str::FromStr;
 
 use crate::analytics::Analytics;
 use crate::extractors::authentication::{policies::*, GuardedData};
 use crate::extractors::sequential_extractor::SeqHandler;
 use crate::task::{TaskListView, TaskStatus, TaskType, TaskView};
 
+use super::{fold_star_or, StarOr};
+
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::get().to(SeqHandler(get_tasks))))
         .service(web::resource("/{task_id}").route(web::get().to(SeqHandler(get_task))));
@@ -27,40 +28,6 @@ pub struct TaskFilterQuery {
     index_uid: Option<CS<StarOr<IndexUid>>>,
 }
 
-/// A type that tries to match either a star (*) or
-/// any other thing that implements `FromStr`.
-#[derive(Debug)]
-enum StarOr<T> {
-    Star,
-    Other(T),
-}
-
-impl<T: FromStr> FromStr for StarOr<T> {
-    type Err = T::Err;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        if s.trim() == "*" {
-            Ok(StarOr::Star)
-        } else {
-            T::from_str(s).map(StarOr::Other)
-        }
-    }
-}
-
-/// Extracts the raw values from the `StarOr` types and
-/// return None if a `StarOr::Star` is encountered.
-fn fold_star_or<T>(content: Vec<StarOr<T>>) -> Option<Vec<T>> {
-    content
-        .into_iter()
-        .fold(Some(Vec::new()), |acc, val| match (acc, val) {
-            (None, _) | (_, StarOr::Star) => None,
-            (Some(mut acc), StarOr::Other(uid)) => {
-                acc.push(uid);
-                Some(acc)
-            }
-        })
-}
-
 #[rustfmt::skip]
 fn task_type_matches_content(type_: &TaskType, content: &TaskContent) -> bool {
     matches!((type_, content),

meilisearch-http/tests/common/index.rs

@@ -145,9 +145,12 @@ impl Index<'_> {
     pub async fn get_document(
         &self,
         id: u64,
-        _options: Option<GetDocumentOptions>,
+        options: Option<GetDocumentOptions>,
     ) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
+        let mut url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
+        if let Some(fields) = options.and_then(|o| o.fields) {
+            url.push_str(&format!("?fields={}", fields.join(",")));
+        }
         self.service.get(url).await
     }
 
@@ -162,10 +165,7 @@ impl Index<'_> {
         }
 
         if let Some(attributes_to_retrieve) = options.attributes_to_retrieve {
-            url.push_str(&format!(
+            url.push_str(&format!("fields={}&", attributes_to_retrieve.join(",")));
-                "attributesToRetrieve={}&",
-                attributes_to_retrieve.join(",")
-            ));
         }
 
         self.service.get(url).await
@@ -245,7 +245,9 @@ impl Index<'_> {
     make_settings_test_routes!(distinct_attribute);
 }
 
-pub struct GetDocumentOptions;
+pub struct GetDocumentOptions {
+    pub fields: Option<Vec<&'static str>>,
+}
 
 #[derive(Debug, Default)]
 pub struct GetAllDocumentsOptions {

meilisearch-http/tests/documents/add_documents.rs

@@ -828,7 +828,7 @@ async fn add_larger_dataset() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 77);
+    assert_eq!(response["results"].as_array().unwrap().len(), 77);
 }
 
 #[actix_rt::test]
@@ -849,7 +849,7 @@ async fn update_larger_dataset() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 77);
+    assert_eq!(response["results"].as_array().unwrap().len(), 77);
 }
 
 #[actix_rt::test]

meilisearch-http/tests/documents/delete_documents.rs

@@ -72,7 +72,7 @@ async fn clear_all_documents() {
         .get_all_documents(GetAllDocumentsOptions::default())
         .await;
     assert_eq!(code, 200);
-    assert!(response.as_array().unwrap().is_empty());
+    assert!(response["results"].as_array().unwrap().is_empty());
 }
 
 #[actix_rt::test]
@@ -89,7 +89,7 @@ async fn clear_all_documents_empty_index() {
         .get_all_documents(GetAllDocumentsOptions::default())
         .await;
     assert_eq!(code, 200);
-    assert!(response.as_array().unwrap().is_empty());
+    assert!(response["results"].as_array().unwrap().is_empty());
 }
 
 #[actix_rt::test]
@@ -125,8 +125,8 @@ async fn delete_batch() {
         .get_all_documents(GetAllDocumentsOptions::default())
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 1);
+    assert_eq!(response["results"].as_array().unwrap().len(), 1);
-    assert_eq!(response.as_array().unwrap()[0]["id"], 3);
+    assert_eq!(response["results"][0]["id"], json!(3));
 }
 
 #[actix_rt::test]
@@ -143,5 +143,5 @@ async fn delete_no_document_batch() {
         .get_all_documents(GetAllDocumentsOptions::default())
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 3);
+    assert_eq!(response["results"].as_array().unwrap().len(), 3);
 }

meilisearch-http/tests/documents/get_documents.rs

@@ -1,5 +1,5 @@
-use crate::common::GetAllDocumentsOptions;
 use crate::common::Server;
+use crate::common::{GetAllDocumentsOptions, GetDocumentOptions};
 
 use serde_json::json;
 
@@ -39,7 +39,7 @@ async fn get_document() {
     let documents = serde_json::json!([
         {
             "id": 0,
-            "content": "foobar",
+            "nested": { "content": "foobar" },
         }
     ]);
     let (_, code) = index.add_documents(documents, None).await;
@@ -49,11 +49,45 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        serde_json::json!(        {
+        serde_json::json!({
             "id": 0,
-            "content": "foobar",
+            "nested": { "content": "foobar" },
         })
     );
+
+    let (response, code) = index
+        .get_document(
+            0,
+            Some(GetDocumentOptions {
+                fields: Some(vec!["id"]),
+            }),
+        )
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(
+        response,
+        serde_json::json!({
+            "id": 0,
+        })
+    );
+
+    /* This currently doesn't work but should be fixed by #2433
+    let (response, code) = index
+        .get_document(
+            0,
+            Some(GetDocumentOptions {
+                fields: Some(vec!["nested.content"]),
+            }),
+        )
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(
+        response,
+        serde_json::json!({
+            "nested": { "content": "foobar" },
+        })
+    );
+    */
 }
 
 #[actix_rt::test]
@@ -88,7 +122,7 @@ async fn get_no_document() {
         .get_all_documents(GetAllDocumentsOptions::default())
         .await;
     assert_eq!(code, 200);
-    assert!(response.as_array().unwrap().is_empty());
+    assert!(response["results"].as_array().unwrap().is_empty());
 }
 
 #[actix_rt::test]
@@ -101,7 +135,7 @@ async fn get_all_documents_no_options() {
         .get_all_documents(GetAllDocumentsOptions::default())
         .await;
     assert_eq!(code, 200);
-    let arr = response.as_array().unwrap();
+    let arr = response["results"].as_array().unwrap();
     assert_eq!(arr.len(), 20);
     let first = serde_json::json!({
         "id":0,
@@ -137,8 +171,11 @@ async fn test_get_all_documents_limit() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 5);
+    assert_eq!(response["results"].as_array().unwrap().len(), 5);
-    assert_eq!(response.as_array().unwrap()[0]["id"], 0);
+    assert_eq!(response["results"][0]["id"], json!(0));
+    assert_eq!(response["offset"], json!(0));
+    assert_eq!(response["limit"], json!(5));
+    assert_eq!(response["total"], json!(77));
 }
 
 #[actix_rt::test]
@@ -154,8 +191,11 @@ async fn test_get_all_documents_offset() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
-    assert_eq!(response.as_array().unwrap()[0]["id"], 5);
+    assert_eq!(response["results"][0]["id"], json!(5));
+    assert_eq!(response["offset"], json!(5));
+    assert_eq!(response["limit"], json!(20));
+    assert_eq!(response["total"], json!(77));
 }
 
 #[actix_rt::test]
@@ -171,20 +211,14 @@ async fn test_get_all_documents_attributes_to_retrieve() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
-    assert_eq!(
+    for results in response["results"].as_array().unwrap() {
-        response.as_array().unwrap()[0]
+        assert_eq!(results.as_object().unwrap().keys().count(), 1);
-            .as_object()
+        assert!(results["name"] != json!(null));
-            .unwrap()
+    }
-            .keys()
+    assert_eq!(response["offset"], json!(0));
-            .count(),
+    assert_eq!(response["limit"], json!(20));
-        1
+    assert_eq!(response["total"], json!(77));
-    );
-    assert!(response.as_array().unwrap()[0]
-        .as_object()
-        .unwrap()
-        .get("name")
-        .is_some());
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
@@ -193,15 +227,13 @@ async fn test_get_all_documents_attributes_to_retrieve() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
-    assert_eq!(
+    for results in response["results"].as_array().unwrap() {
-        response.as_array().unwrap()[0]
+        assert_eq!(results.as_object().unwrap().keys().count(), 0);
-            .as_object()
+    }
-            .unwrap()
+    assert_eq!(response["offset"], json!(0));
-            .keys()
+    assert_eq!(response["limit"], json!(20));
-            .count(),
+    assert_eq!(response["total"], json!(77));
-        0
-    );
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
@@ -210,15 +242,13 @@ async fn test_get_all_documents_attributes_to_retrieve() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
-    assert_eq!(
+    for results in response["results"].as_array().unwrap() {
-        response.as_array().unwrap()[0]
+        assert_eq!(results.as_object().unwrap().keys().count(), 0);
-            .as_object()
+    }
-            .unwrap()
+    assert_eq!(response["offset"], json!(0));
-            .keys()
+    assert_eq!(response["limit"], json!(20));
-            .count(),
+    assert_eq!(response["total"], json!(77));
-        0
-    );
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
@@ -227,15 +257,12 @@ async fn test_get_all_documents_attributes_to_retrieve() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
-    assert_eq!(
+    for results in response["results"].as_array().unwrap() {
-        response.as_array().unwrap()[0]
+        assert_eq!(results.as_object().unwrap().keys().count(), 2);
-            .as_object()
+        assert!(results["name"] != json!(null));
-            .unwrap()
+        assert!(results["tags"] != json!(null));
-            .keys()
+    }
-            .count(),
-        2
-    );
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
@@ -244,15 +271,10 @@ async fn test_get_all_documents_attributes_to_retrieve() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
-    assert_eq!(
+    for results in response["results"].as_array().unwrap() {
-        response.as_array().unwrap()[0]
+        assert_eq!(results.as_object().unwrap().keys().count(), 16);
-            .as_object()
+    }
-            .unwrap()
-            .keys()
-            .count(),
-        16
-    );
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
@@ -261,19 +283,14 @@ async fn test_get_all_documents_attributes_to_retrieve() {
         })
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
-    assert_eq!(
+    for results in response["results"].as_array().unwrap() {
-        response.as_array().unwrap()[0]
+        assert_eq!(results.as_object().unwrap().keys().count(), 16);
-            .as_object()
+    }
-            .unwrap()
-            .keys()
-            .count(),
-        16
-    );
 }
 
 #[actix_rt::test]
-async fn get_documents_displayed_attributes() {
+async fn get_documents_displayed_attributes_is_ignored() {
     let server = Server::new().await;
     let index = server.index("test");
     index
@@ -285,23 +302,19 @@ async fn get_documents_displayed_attributes() {
         .get_all_documents(GetAllDocumentsOptions::default())
         .await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
     assert_eq!(
-        response.as_array().unwrap()[0]
+        response["results"][0].as_object().unwrap().keys().count(),
-            .as_object()
+        16
-            .unwrap()
-            .keys()
-            .count(),
-        1
     );
-    assert!(response.as_array().unwrap()[0]
+    assert!(response["results"][0]["gender"] != json!(null));
-        .as_object()
+
-        .unwrap()
+    assert_eq!(response["offset"], json!(0));
-        .get("gender")
+    assert_eq!(response["limit"], json!(20));
-        .is_some());
+    assert_eq!(response["total"], json!(77));
 
     let (response, code) = index.get_document(0, None).await;
     assert_eq!(code, 200);
-    assert_eq!(response.as_object().unwrap().keys().count(), 1);
+    assert_eq!(response.as_object().unwrap().keys().count(), 16);
     assert!(response.as_object().unwrap().get("gender").is_some());
 }

meilisearch-http/tests/dumps/mod.rs

@@ -142,21 +142,21 @@ async fn import_dump_v2_movie_with_settings() {
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({ "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 })
+        json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 })
     );
 
     let (document, code) = index.get_document(500, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000})
+        json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000})
     );
 
     let (document, code) = index.get_document(10006, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600})
+        json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600})
     );
 }
 
@@ -211,21 +211,21 @@ async fn import_dump_v2_rubygems_with_settings() {
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "version": "0.15.2", "total_downloads": "7465"})
+        json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"})
     );
 
     let (document, code) = index.get_document(191940, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "version": "1.1.0", "total_downloads": "9394"})
+        json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"})
     );
 
     let (document, code) = index.get_document(159227, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "version": "0.1.0", "total_downloads": "1007"})
+        json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"})
     );
 }
 
@@ -341,21 +341,21 @@ async fn import_dump_v3_movie_with_settings() {
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({ "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 })
+        json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 })
     );
 
     let (document, code) = index.get_document(500, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000})
+        json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000})
     );
 
     let (document, code) = index.get_document(10006, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600})
+        json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600})
     );
 }
 
@@ -410,21 +410,21 @@ async fn import_dump_v3_rubygems_with_settings() {
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "version": "0.15.2", "total_downloads": "7465"})
+        json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"})
     );
 
     let (document, code) = index.get_document(191940, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "version": "1.1.0", "total_downloads": "9394"})
+        json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"})
     );
 
     let (document, code) = index.get_document(159227, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({"name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "version": "0.1.0", "total_downloads": "1007"})
+        json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"})
     );
 }
 
@@ -540,21 +540,21 @@ async fn import_dump_v4_movie_with_settings() {
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({ "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 })
+        json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 })
     );
 
     let (document, code) = index.get_document(500, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({ "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000})
+        json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000})
     );
 
     let (document, code) = index.get_document(10006, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({ "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600})
+        json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600})
     );
 }
 
@@ -609,20 +609,20 @@ async fn import_dump_v4_rubygems_with_settings() {
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "version": "0.15.2", "total_downloads": "7465"})
+        json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"})
     );
 
     let (document, code) = index.get_document(191940, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "version": "1.1.0", "total_downloads": "9394"})
+        json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"})
     );
 
     let (document, code) = index.get_document(159227, None).await;
     assert_eq!(code, 200);
     assert_eq!(
         document,
-        json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "version": "0.1.0", "total_downloads": "1007"})
+        json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"})
     );
 }

meilisearch-lib/src/index/index.rs

@@ -1,4 +1,4 @@
-use std::collections::{BTreeSet, HashSet};
+use std::collections::BTreeSet;
 use std::fs::create_dir_all;
 use std::marker::PhantomData;
 use std::ops::Deref;
@@ -218,17 +218,17 @@ impl Index {
         })
     }
 
+    /// Return the total number of documents contained in the index + the selected documents.
     pub fn retrieve_documents<S: AsRef<str>>(
         &self,
         offset: usize,
         limit: usize,
         attributes_to_retrieve: Option<Vec<S>>,
-    ) -> Result<Vec<Map<String, Value>>> {
+    ) -> Result<(u64, Vec<Document>)> {
         let txn = self.read_txn()?;
 
         let fields_ids_map = self.fields_ids_map(&txn)?;
-        let fields_to_display =
+        let fields_to_display = self.fields_to_display(&attributes_to_retrieve, &fields_ids_map)?;
-            self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?;
 
         let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit);
 
@@ -240,20 +240,20 @@ impl Index {
             documents.push(object);
         }
 
-        Ok(documents)
+        let number_of_documents = self.number_of_documents(&txn)?;
+
+        Ok((number_of_documents, documents))
     }
 
     pub fn retrieve_document<S: AsRef<str>>(
         &self,
         doc_id: String,
         attributes_to_retrieve: Option<Vec<S>>,
-    ) -> Result<Map<String, Value>> {
+    ) -> Result<Document> {
         let txn = self.read_txn()?;
 
         let fields_ids_map = self.fields_ids_map(&txn)?;
-
+        let fields_to_display = self.fields_to_display(&attributes_to_retrieve, &fields_ids_map)?;
-        let fields_to_display =
-            self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?;
 
         let internal_id = self
             .external_documents_ids(&txn)?
@@ -278,25 +278,18 @@ impl Index {
 
     fn fields_to_display<S: AsRef<str>>(
         &self,
-        txn: &milli::heed::RoTxn,
         attributes_to_retrieve: &Option<Vec<S>>,
         fields_ids_map: &milli::FieldsIdsMap,
     ) -> Result<Vec<FieldId>> {
-        let mut displayed_fields_ids = match self.displayed_fields_ids(txn)? {
-            Some(ids) => ids.into_iter().collect::<Vec<_>>(),
-            None => fields_ids_map.iter().map(|(id, _)| id).collect(),
-        };
-
         let attributes_to_retrieve_ids = match attributes_to_retrieve {
             Some(attrs) => attrs
                 .iter()
                 .filter_map(|f| fields_ids_map.id(f.as_ref()))
-                .collect::<HashSet<_>>(),
+                .collect(),
             None => fields_ids_map.iter().map(|(id, _)| id).collect(),
         };
 
-        displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid));
+        Ok(attributes_to_retrieve_ids)
-        Ok(displayed_fields_ids)
     }
 
     pub fn snapshot(&self, path: impl AsRef<Path>) -> Result<()> {

meilisearch-lib/src/index/mod.rs

@@ -32,11 +32,11 @@ pub mod test {
     use milli::update::IndexerConfig;
     use milli::update::{DocumentAdditionResult, DocumentDeletionResult, IndexDocumentsMethod};
     use nelson::Mocker;
-    use serde_json::{Map, Value};
     use uuid::Uuid;
 
     use super::error::Result;
     use super::index::Index;
+    use super::Document;
     use super::{Checked, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings};
     use crate::update_file_store::UpdateFileStore;
 
@@ -102,7 +102,7 @@ pub mod test {
             offset: usize,
             limit: usize,
             attributes_to_retrieve: Option<Vec<S>>,
-        ) -> Result<Vec<Map<String, Value>>> {
+        ) -> Result<(u64, Vec<Document>)> {
             match self {
                 MockIndex::Real(index) => {
                     index.retrieve_documents(offset, limit, attributes_to_retrieve)
@@ -115,7 +115,7 @@ pub mod test {
             &self,
             doc_id: String,
             attributes_to_retrieve: Option<Vec<S>>,
-        ) -> Result<Map<String, Value>> {
+        ) -> Result<Document> {
             match self {
                 MockIndex::Real(index) => index.retrieve_document(doc_id, attributes_to_retrieve),
                 MockIndex::Mock(_) => todo!(),

meilisearch-lib/src/index_controller/mod.rs

@@ -524,18 +524,19 @@ where
         Ok(settings)
     }
 
+    /// Return the total number of documents contained in the index + the selected documents.
     pub async fn documents(
         &self,
         uid: String,
         offset: usize,
         limit: usize,
         attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Document>> {
+    ) -> Result<(u64, Vec<Document>)> {
         let index = self.index_resolver.get_index(uid).await?;
-        let documents =
+        let result =
             spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve))
                 .await??;
-        Ok(documents)
+        Ok(result)
     }
 
     pub async fn document(