From 250aeaa86cd76e7357a72b9227a87f3c27c47b69 Mon Sep 17 00:00:00 2001
From: qdequele <quentin@dequelen.me>
Date: Thu, 27 Feb 2020 14:51:29 +0100
Subject: [PATCH] stop reindexing by chunk during complete reindexing

---
 .../src/update/documents_addition.rs          | 71 +++++++++----------
 1 file changed, 34 insertions(+), 37 deletions(-)

diff --git a/meilisearch-core/src/update/documents_addition.rs b/meilisearch-core/src/update/documents_addition.rs
index 0513d3e0c..2a2fe978a 100644
--- a/meilisearch-core/src/update/documents_addition.rs
+++ b/meilisearch-core/src/update/documents_addition.rs
@@ -279,49 +279,46 @@ pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Ind
     index.postings_lists.clear(writer)?;
     index.docs_words.clear(writer)?;
 
-    // 3. re-index chunks of documents (otherwise we make the borrow checker unhappy)
-    for documents_ids in documents_ids_to_reindex.chunks(100) {
-        let stop_words = match index.main.stop_words_fst(writer)? {
-            Some(stop_words) => stop_words,
-            None => fst::Set::default(),
-        };
+    let stop_words = match index.main.stop_words_fst(writer)? {
+        Some(stop_words) => stop_words,
+        None => fst::Set::default(),
+    };
 
-        let number_of_inserted_documents = documents_ids.len();
-        let mut indexer = RawIndexer::new(stop_words);
-        let mut ram_store = HashMap::new();
+    let number_of_inserted_documents = documents_ids_to_reindex.len();
+    let mut indexer = RawIndexer::new(stop_words);
+    let mut ram_store = HashMap::new();
 
-        for document_id in documents_ids {
-            for result in index.documents_fields.document_fields(writer, *document_id)? {
-                let (field_id, bytes) = result?;
-                let value: serde_json::Value = serde_json::from_slice(bytes)?;
-                ram_store.insert((document_id, field_id), value);
-            }
-
-            for ((docid, field_id), value) in ram_store.drain() {
-                serialize_value_with_id(
-                    writer,
-                    field_id,
-                    &schema,
-                    *docid,
-                    index.documents_fields,
-                    index.documents_fields_counts,
-                    &mut indexer,
-                    &mut ranked_map,
-                    &value
-                )?;
-            }
+    for document_id in documents_ids_to_reindex {
+        for result in index.documents_fields.document_fields(writer, document_id)? {
+            let (field_id, bytes) = result?;
+            let value: serde_json::Value = serde_json::from_slice(bytes)?;
+            ram_store.insert((document_id, field_id), value);
         }
 
-        // 4. write the new index in the main store
-        write_documents_addition_index(
-            writer,
-            index,
-            &ranked_map,
-            number_of_inserted_documents,
-            indexer,
-        )?;
+        for ((docid, field_id), value) in ram_store.drain() {
+            serialize_value_with_id(
+                writer,
+                field_id,
+                &schema,
+                docid,
+                index.documents_fields,
+                index.documents_fields_counts,
+                &mut indexer,
+                &mut ranked_map,
+                &value
+            )?;
+        }
     }
 
+    // 4. write the new index in the main store
+    write_documents_addition_index(
+        writer,
+        index,
+        &ranked_map,
+        number_of_inserted_documents,
+        indexer,
+    )?;
+
     index.main.put_schema(writer, &schema)?;
 
     Ok(())