From 24e84d7ca176519ec838ff27d4b7ed9d0faaef38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= <clementine@meilisearch.com>
Date: Mon, 16 Aug 2021 13:43:06 +0200
Subject: [PATCH 1/7] Test new indexer

---
 Cargo.lock                                   | 68 ++++++++++++++++++--
 meilisearch-http/Cargo.toml                  |  2 +-
 meilisearch-http/src/index/update_handler.rs |  2 +-
 3 files changed, 64 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e92c0ed0f..eddb07937 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -458,6 +458,20 @@ name = "bytemuck"
 version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9966d2ab714d0f785dbac0a0396251a35280aeb42413281617d0209ab4898435"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e215f8c2f9f79cb53c8335e687ffd07d5bfcb6fe5fc80723762d0be46e7cc54"
+dependencies = [
+ "proc-macro2 1.0.27",
+ "quote 1.0.9",
+ "syn 1.0.73",
+]
 
 [[package]]
 name = "byteorder"
@@ -630,6 +644,12 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
+
 [[package]]
 name = "cow-utils"
 version = "0.1.2"
@@ -787,6 +807,12 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0"
 
+[[package]]
+name = "doc-comment"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
+
 [[package]]
 name = "downcast"
 version = "0.10.0"
@@ -1097,9 +1123,11 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
 [[package]]
 name = "grenad"
-version = "0.1.0"
-source = "git+https://github.com/Kerollmops/grenad.git?rev=3adcb26#3adcb267dcbc590c7da10eb5f887a254865b3dbe"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29a631f65b886f839c561fc5d086ba0fb1a27df5bb52f57439e415238329ce08"
 dependencies = [
+ "bytemuck",
  "byteorder",
  "flate2",
  "log",
@@ -1594,7 +1622,7 @@ dependencies = [
  "log",
  "main_error",
  "meilisearch-error",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.4",
  "memmap",
  "milli",
  "mime",
@@ -1645,7 +1673,7 @@ dependencies = [
  "once_cell",
  "slice-group-by",
  "unicode-segmentation",
- "whatlang",
+ "whatlang 0.12.0",
 ]
 
 [[package]]
@@ -1676,12 +1704,14 @@ dependencies = [
 [[package]]
 name = "milli"
 version = "0.11.0"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.11.0#c51bb6789cb3fbb6511138374b3443f9116a445c"
+source = "git+https://github.com/meilisearch/milli.git?branch=plug-new-indexer#a1dc915af8d62c77f0431bb1d0fc4e53cc35ff42"
 dependencies = [
  "bstr",
+ "byte-unit",
  "byteorder",
  "chrono",
  "concat-arrays",
+ "crossbeam-channel",
  "csv",
  "either",
  "flate2",
@@ -1695,7 +1725,7 @@ dependencies = [
  "linked-hash-map",
  "log",
  "logging_timer",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.3",
  "memmap",
  "obkv",
  "once_cell",
@@ -1709,6 +1739,7 @@ dependencies = [
  "slice-group-by",
  "smallstr",
  "smallvec",
+ "sysinfo",
  "tempfile",
  "uuid",
 ]
@@ -2812,6 +2843,22 @@ dependencies = [
  "unicode-xid 0.2.2",
 ]
 
+[[package]]
+name = "sysinfo"
+version = "0.19.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e7de153d0438a648bb71e06e300e54fc641685e96af96d49b843f43172d341c"
+dependencies = [
+ "cfg-if 1.0.0",
+ "core-foundation-sys",
+ "doc-comment",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "rayon",
+ "winapi",
+]
+
 [[package]]
 name = "tar"
 version = "0.4.35"
@@ -3319,6 +3366,15 @@ dependencies = [
  "webpki",
 ]
 
+[[package]]
+name = "whatlang"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075"
+dependencies = [
+ "hashbrown 0.7.2",
+]
+
 [[package]]
 name = "whatlang"
 version = "0.12.0"
diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml
index f8a56013c..ecb1d9992 100644
--- a/meilisearch-http/Cargo.toml
+++ b/meilisearch-http/Cargo.toml
@@ -50,7 +50,7 @@ main_error = "0.1.0"
 meilisearch-error = { path = "../meilisearch-error" }
 meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
 memmap = "0.7.0"
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.11.0" }
+milli = { git = "https://github.com/meilisearch/milli.git", branch = "plug-new-indexer" }
 mime = "0.3.16"
 num_cpus = "1.13.0"
 once_cell = "1.5.2"
diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs
index 1426cca01..2017a9fcf 100644
--- a/meilisearch-http/src/index/update_handler.rs
+++ b/meilisearch-http/src/index/update_handler.rs
@@ -1,8 +1,8 @@
 use std::fs::File;
 
 use crate::index::Index;
-use milli::update::UpdateBuilder;
 use milli::CompressionType;
+use milli::update::UpdateBuilder;
 use rayon::ThreadPool;
 
 use crate::index_controller::UpdateMeta;

From b092a624eddeb176e6c1c68daf6c230a306f3002 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <clement@meilisearch.com>
Date: Thu, 2 Sep 2021 18:18:59 +0200
Subject: [PATCH 2/7] Introduce the MaxMemory struct that defaults to 2/3 of
 the available memory

---
 Cargo.lock                                   | 18 ++++-
 meilisearch-http/Cargo.toml                  |  1 +
 meilisearch-http/src/index/update_handler.rs | 11 +--
 meilisearch-http/src/option.rs               | 72 ++++++++++++++++++--
 4 files changed, 90 insertions(+), 12 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index eddb07937..b1a495f82 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1647,6 +1647,7 @@ dependencies = [
  "siphasher",
  "slice-group-by",
  "structopt",
+ "sysinfo 0.20.0",
  "tar",
  "tempdir",
  "tempfile",
@@ -1739,7 +1740,7 @@ dependencies = [
  "slice-group-by",
  "smallstr",
  "smallvec",
- "sysinfo",
+ "sysinfo 0.19.2",
  "tempfile",
  "uuid",
 ]
@@ -2859,6 +2860,21 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "sysinfo"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0af066e6272f2175c1783cfc2ebf3e2d8dfe2c182b00677fdeccbf8291af83fb"
+dependencies = [
+ "cfg-if 1.0.0",
+ "core-foundation-sys",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "rayon",
+ "winapi",
+]
+
 [[package]]
 name = "tar"
 version = "0.4.35"
diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml
index ecb1d9992..e845edc13 100644
--- a/meilisearch-http/Cargo.toml
+++ b/meilisearch-http/Cargo.toml
@@ -76,6 +76,7 @@ pin-project = "1.0.7"
 whoami = { version = "1.1.2", optional = true }
 reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true }
 serdeval = "0.1.0"
+sysinfo = "0.20.0"
 
 [dev-dependencies]
 actix-rt = "2.1.0"
diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs
index 2017a9fcf..0bca4f9b8 100644
--- a/meilisearch-http/src/index/update_handler.rs
+++ b/meilisearch-http/src/index/update_handler.rs
@@ -1,8 +1,8 @@
 use std::fs::File;
 
 use crate::index::Index;
-use milli::CompressionType;
 use milli::update::UpdateBuilder;
+use milli::CompressionType;
 use rayon::ThreadPool;
 
 use crate::index_controller::UpdateMeta;
@@ -14,7 +14,7 @@ pub struct UpdateHandler {
     chunk_compression_level: Option<u32>,
     thread_pool: ThreadPool,
     log_frequency: usize,
-    max_memory: usize,
+    max_memory: Option<usize>,
     linked_hash_map_size: usize,
     chunk_compression_type: CompressionType,
     chunk_fusing_shrink_size: u64,
@@ -25,12 +25,13 @@ impl UpdateHandler {
         let thread_pool = rayon::ThreadPoolBuilder::new()
             .num_threads(opt.indexing_jobs.unwrap_or(num_cpus::get() / 2))
             .build()?;
+
         Ok(Self {
             max_nb_chunks: opt.max_nb_chunks,
             chunk_compression_level: opt.chunk_compression_level,
             thread_pool,
             log_frequency: opt.log_every_n,
-            max_memory: opt.max_memory.get_bytes() as usize,
+            max_memory: opt.max_memory.map(|m| m.get_bytes() as usize),
             linked_hash_map_size: opt.linked_hash_map_size,
             chunk_compression_type: opt.chunk_compression_type,
             chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(),
@@ -48,7 +49,9 @@ impl UpdateHandler {
         }
         update_builder.thread_pool(&self.thread_pool);
         update_builder.log_every_n(self.log_frequency);
-        update_builder.max_memory(self.max_memory);
+        if let Some(max_memory) = self.max_memory {
+            update_builder.max_memory(max_memory);
+        }
         update_builder.linked_hash_map_size(self.linked_hash_map_size);
         update_builder.chunk_compression_type(self.chunk_compression_type);
         update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size);
diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs
index d87b98adb..14c9e5a13 100644
--- a/meilisearch-http/src/option.rs
+++ b/meilisearch-http/src/option.rs
@@ -1,5 +1,9 @@
+use byte_unit::ByteError;
+use std::fmt;
 use std::io::{BufReader, Read};
+use std::ops::Deref;
 use std::path::PathBuf;
+use std::str::FromStr;
 use std::sync::Arc;
 use std::{error, fs};
 
@@ -11,6 +15,7 @@ use rustls::{
     RootCertStore,
 };
 use structopt::StructOpt;
+use sysinfo::{RefreshKind, System, SystemExt};
 
 #[derive(Debug, Clone, StructOpt)]
 pub struct IndexerOpts {
@@ -23,13 +28,15 @@ pub struct IndexerOpts {
     #[structopt(long)]
     pub max_nb_chunks: Option<usize>,
 
-    /// The maximum amount of memory to use for the Grenad buffer. It is recommended
-    /// to use something like 80%-90% of the available memory.
+    /// The maximum amount of memory the indexer will use. It defaults to 2/3
+    /// of the available memory. It is recommended to use something like 80%-90%
+    /// of the available memory, no more.
     ///
-    /// It is automatically split by the number of jobs e.g. if you use 7 jobs
-    /// and 7 GB of max memory, each thread will use a maximum of 1 GB.
-    #[structopt(long, default_value = "7 GiB")]
-    pub max_memory: Byte,
+    /// In case the engine is unable to retrieve the available memory the engine will
+    /// try to use the memory it needs but without real limit, this can lead to
+    /// Out-Of-Memory issues and it is recommended to specify the amount of memory to use.
+    #[structopt(long)]
+    pub max_memory: MaxMemory,
 
     /// Size of the linked hash map cache when indexing.
     /// The bigger it is, the faster the indexing is but the more memory it takes.
@@ -69,7 +76,7 @@ impl Default for IndexerOpts {
         Self {
             log_every_n: 100_000,
             max_nb_chunks: None,
-            max_memory: Byte::from_str("1GiB").unwrap(),
+            max_memory: MaxMemory::default(),
             linked_hash_map_size: 500,
             chunk_compression_type: CompressionType::None,
             chunk_compression_level: None,
@@ -240,6 +247,57 @@ impl Opt {
     }
 }
 
+/// A type used to detect the max memory available and use 2/3 of it.
+#[derive(Debug, Clone, Copy)]
+pub struct MaxMemory(Option<Byte>);
+
+impl FromStr for MaxMemory {
+    type Err = ByteError;
+
+    fn from_str(s: &str) -> Result<MaxMemory, ByteError> {
+        Byte::from_str(s).map(Some).map(MaxMemory)
+    }
+}
+
+impl Default for MaxMemory {
+    fn default() -> MaxMemory {
+        MaxMemory(
+            total_memory_bytes()
+                .map(|bytes| bytes * 2 / 3)
+                .map(Byte::from_bytes),
+        )
+    }
+}
+
+impl fmt::Display for MaxMemory {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self.0 {
+            Some(memory) => write!(f, "{}", memory.get_appropriate_unit(true)),
+            None => f.write_str("unknown"),
+        }
+    }
+}
+
+impl Deref for MaxMemory {
+    type Target = Option<Byte>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+/// Returns the total amount of bytes available or `None` if this system isn't supported.
+fn total_memory_bytes() -> Option<u64> {
+    if System::IS_SUPPORTED {
+        let memory_kind = RefreshKind::new().with_memory();
+        let mut system = System::new_with_specifics(memory_kind);
+        system.refresh_memory();
+        Some(system.total_memory() * 1024) // KiB into bytes
+    } else {
+        None
+    }
+}
+
 fn load_certs(filename: PathBuf) -> Result<Vec<rustls::Certificate>, Box<dyn error::Error>> {
     let certfile = fs::File::open(filename).map_err(|_| "cannot open certificate file")?;
     let mut reader = BufReader::new(certfile);

From 0cd66c3a89576f41b97fab8f4d2e0d1a11841bec Mon Sep 17 00:00:00 2001
From: Kerollmops <clement@meilisearch.com>
Date: Mon, 16 Aug 2021 18:45:58 +0200
Subject: [PATCH 3/7] Bump the milli version

---
 Cargo.lock                                   | 87 ++++++++------------
 meilisearch-http/src/index/update_handler.rs |  6 --
 2 files changed, 33 insertions(+), 60 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b1a495f82..47db33f78 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -807,12 +807,6 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0"
 
-[[package]]
-name = "doc-comment"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
-
 [[package]]
 name = "downcast"
 version = "0.10.0"
@@ -1123,15 +1117,14 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
 [[package]]
 name = "grenad"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29a631f65b886f839c561fc5d086ba0fb1a27df5bb52f57439e415238329ce08"
+checksum = "7824d499230110f4e4a8d4fd3fd4dc15c1347fce5082e4bba82eef17f43e1ed8"
 dependencies = [
  "bytemuck",
  "byteorder",
  "flate2",
- "log",
- "nix",
+ "lz4_flex",
  "snap",
  "tempfile",
  "zstd",
@@ -1562,6 +1555,15 @@ dependencies = [
  "syn 0.15.44",
 ]
 
+[[package]]
+name = "lz4_flex"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827b976d911b5d2e42b2ccfc7c0d2461a1414e8280436885218762fc529b3f8"
+dependencies = [
+ "twox-hash",
+]
+
 [[package]]
 name = "main_error"
 version = "0.1.1"
@@ -1622,7 +1624,7 @@ dependencies = [
  "log",
  "main_error",
  "meilisearch-error",
- "meilisearch-tokenizer 0.2.4",
+ "meilisearch-tokenizer",
  "memmap",
  "milli",
  "mime",
@@ -1647,7 +1649,7 @@ dependencies = [
  "siphasher",
  "slice-group-by",
  "structopt",
- "sysinfo 0.20.0",
+ "sysinfo",
  "tar",
  "tempdir",
  "tempfile",
@@ -1674,7 +1676,7 @@ dependencies = [
  "once_cell",
  "slice-group-by",
  "unicode-segmentation",
- "whatlang 0.12.0",
+ "whatlang",
 ]
 
 [[package]]
@@ -1704,8 +1706,7 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "0.11.0"
-source = "git+https://github.com/meilisearch/milli.git?branch=plug-new-indexer#a1dc915af8d62c77f0431bb1d0fc4e53cc35ff42"
+
 dependencies = [
  "bstr",
  "byte-unit",
@@ -1726,7 +1727,7 @@ dependencies = [
  "linked-hash-map",
  "log",
  "logging_timer",
- "meilisearch-tokenizer 0.2.3",
+ "meilisearch-tokenizer",
  "memmap",
  "obkv",
  "once_cell",
@@ -1740,7 +1741,6 @@ dependencies = [
  "slice-group-by",
  "smallstr",
  "smallvec",
- "sysinfo 0.19.2",
  "tempfile",
  "uuid",
 ]
@@ -1820,18 +1820,6 @@ dependencies = [
  "syn 1.0.73",
 ]
 
-[[package]]
-name = "nix"
-version = "0.19.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2"
-dependencies = [
- "bitflags",
- "cc",
- "cfg-if 1.0.0",
- "libc",
-]
-
 [[package]]
 name = "normalize-line-endings"
 version = "0.3.0"
@@ -2722,6 +2710,12 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "stdweb"
 version = "0.4.20"
@@ -2844,22 +2838,6 @@ dependencies = [
  "unicode-xid 0.2.2",
 ]
 
-[[package]]
-name = "sysinfo"
-version = "0.19.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e7de153d0438a648bb71e06e300e54fc641685e96af96d49b843f43172d341c"
-dependencies = [
- "cfg-if 1.0.0",
- "core-foundation-sys",
- "doc-comment",
- "libc",
- "ntapi",
- "once_cell",
- "rayon",
- "winapi",
-]
-
 [[package]]
 name = "sysinfo"
 version = "0.20.0"
@@ -3115,6 +3093,16 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
+[[package]]
+name = "twox-hash"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f559b464de2e2bdabcac6a210d12e9b5a5973c251e102c44c585c71d51bd78e"
+dependencies = [
+ "cfg-if 1.0.0",
+ "static_assertions",
+]
+
 [[package]]
 name = "typenum"
 version = "1.13.0"
@@ -3382,15 +3370,6 @@ dependencies = [
  "webpki",
 ]
 
-[[package]]
-name = "whatlang"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075"
-dependencies = [
- "hashbrown 0.7.2",
-]
-
 [[package]]
 name = "whatlang"
 version = "0.12.0"
diff --git a/meilisearch-http/src/index/update_handler.rs b/meilisearch-http/src/index/update_handler.rs
index 0bca4f9b8..f3977a00d 100644
--- a/meilisearch-http/src/index/update_handler.rs
+++ b/meilisearch-http/src/index/update_handler.rs
@@ -15,9 +15,7 @@ pub struct UpdateHandler {
     thread_pool: ThreadPool,
     log_frequency: usize,
     max_memory: Option<usize>,
-    linked_hash_map_size: usize,
     chunk_compression_type: CompressionType,
-    chunk_fusing_shrink_size: u64,
 }
 
 impl UpdateHandler {
@@ -32,9 +30,7 @@ impl UpdateHandler {
             thread_pool,
             log_frequency: opt.log_every_n,
             max_memory: opt.max_memory.map(|m| m.get_bytes() as usize),
-            linked_hash_map_size: opt.linked_hash_map_size,
             chunk_compression_type: opt.chunk_compression_type,
-            chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(),
         })
     }
 
@@ -52,9 +48,7 @@ impl UpdateHandler {
         if let Some(max_memory) = self.max_memory {
             update_builder.max_memory(max_memory);
         }
-        update_builder.linked_hash_map_size(self.linked_hash_map_size);
         update_builder.chunk_compression_type(self.chunk_compression_type);
-        update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size);
         update_builder
     }
 

From 63e67f72e3a31829b71d4f224e0d9ba840f3cab7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= <clementine@meilisearch.com>
Date: Wed, 18 Aug 2021 14:24:01 +0200
Subject: [PATCH 4/7] Update tokenizer and new milli version

---
 Cargo.lock | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 47db33f78..b9b9443d0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1706,7 +1706,8 @@ dependencies = [
 
 [[package]]
 name = "milli"
-
+version = "0.10.1"
+source = "git+https://github.com/meilisearch/milli.git?branch=plug-new-indexer#e37100bd7393880eec2571089711f79b71849eb5"
 dependencies = [
  "bstr",
  "byte-unit",

From 4cdf680a8116d9aa36652d9c1c54caae1b1968a4 Mon Sep 17 00:00:00 2001
From: Kerollmops <clement@meilisearch.com>
Date: Tue, 24 Aug 2021 11:46:37 +0200
Subject: [PATCH 5/7] Make the MaxMemory use the default value when undefined

---
 meilisearch-http/src/option.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs
index 14c9e5a13..f3c077c05 100644
--- a/meilisearch-http/src/option.rs
+++ b/meilisearch-http/src/option.rs
@@ -35,7 +35,7 @@ pub struct IndexerOpts {
     /// In case the engine is unable to retrieve the available memory the engine will
     /// try to use the memory it needs but without real limit, this can lead to
     /// Out-Of-Memory issues and it is recommended to specify the amount of memory to use.
-    #[structopt(long)]
+    #[structopt(long, default_value)]
     pub max_memory: MaxMemory,
 
     /// Size of the linked hash map cache when indexing.

From 8d4723d91b955859ac3ff06c9fcfcf7ab66caad1 Mon Sep 17 00:00:00 2001
From: many <maxime@meilisearch.com>
Date: Tue, 31 Aug 2021 14:27:45 +0200
Subject: [PATCH 6/7] Update lock file

---
 Cargo.lock | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b9b9443d0..93297c657 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1182,8 +1182,8 @@ dependencies = [
 
 [[package]]
 name = "heed"
-version = "0.12.0"
-source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
+version = "0.12.1"
+source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#fc017cf3394af737f92fd71e16f0499a78b79d65"
 dependencies = [
  "byteorder",
  "heed-traits",
@@ -1201,12 +1201,12 @@ dependencies = [
 [[package]]
 name = "heed-traits"
 version = "0.7.0"
-source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
+source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#fc017cf3394af737f92fd71e16f0499a78b79d65"
 
 [[package]]
 name = "heed-types"
 version = "0.7.2"
-source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
+source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#fc017cf3394af737f92fd71e16f0499a78b79d65"
 dependencies = [
  "bincode",
  "heed-traits",
@@ -1706,8 +1706,8 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "0.10.1"
-source = "git+https://github.com/meilisearch/milli.git?branch=plug-new-indexer#e37100bd7393880eec2571089711f79b71849eb5"
+version = "0.11.0"
+source = "git+https://github.com/meilisearch/milli.git?branch=plug-new-indexer#4860fd452965d234a11cc8430309bd9782a21bfd"
 dependencies = [
  "bstr",
  "byte-unit",

From 7e80337e5b7b53862e71e390c5c3084608488855 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <clement@meilisearch.com>
Date: Thu, 2 Sep 2021 18:09:03 +0200
Subject: [PATCH 7/7] Bump milli to v0.12.0

---
 Cargo.lock                  | 5 ++---
 meilisearch-http/Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 93297c657..3802c6b73 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1706,11 +1706,10 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "0.11.0"
-source = "git+https://github.com/meilisearch/milli.git?branch=plug-new-indexer#4860fd452965d234a11cc8430309bd9782a21bfd"
+version = "0.12.0"
+source = "git+https://github.com/meilisearch/milli.git?tag=v0.12.0#5cbe8793251bbf143434c8a4c4e7195ca6c5f2ac"
 dependencies = [
  "bstr",
- "byte-unit",
  "byteorder",
  "chrono",
  "concat-arrays",
diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml
index e845edc13..e94d67095 100644
--- a/meilisearch-http/Cargo.toml
+++ b/meilisearch-http/Cargo.toml
@@ -50,7 +50,7 @@ main_error = "0.1.0"
 meilisearch-error = { path = "../meilisearch-error" }
 meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
 memmap = "0.7.0"
-milli = { git = "https://github.com/meilisearch/milli.git", branch = "plug-new-indexer" }
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.12.0" }
 mime = "0.3.16"
 num_cpus = "1.13.0"
 once_cell = "1.5.2"