Merge #2244

2244: chore(all): bump milli r=curquiza a=MarinPostma

continues the work initiated by `@psvnlsaikumar` in #2228

Co-authored-by: Sai Kumar <psvnlsaikumar@gmail.com>

meilisearch-auth/Cargo.toml

@@ -5,7 +5,6 @@ edition = "2021"
 
 [dependencies]
 enum-iterator = "0.7.0"
-heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
 sha2 = "0.9.6"
 meilisearch-error = { path = "../meilisearch-error" }
 serde_json = { version = "1.0.67", features = ["preserve_order"] }
@@ -13,3 +12,4 @@ time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsi
 rand = "0.8.4"
 serde = { version = "1.0.130", features = ["derive"] }
 thiserror = "1.0.28"
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.24.0" }

meilisearch-auth/src/error.rs

@@ -25,7 +25,7 @@ pub enum AuthControllerError {
 }
 
 internal_error!(
-    AuthControllerError: heed::Error,
+    AuthControllerError: milli::heed::Error,
     std::io::Error,
     serde_json::Error,
     std::str::Utf8Error

meilisearch-auth/src/store.rs

@@ -8,8 +8,8 @@ use std::path::Path;
 use std::str;
 use std::sync::Arc;
 
-use heed::types::{ByteSlice, DecodeIgnore, SerdeJson};
+use milli::heed::types::{ByteSlice, DecodeIgnore, SerdeJson};
-use heed::{Database, Env, EnvOpenOptions, RwTxn};
+use milli::heed::{Database, Env, EnvOpenOptions, RwTxn};
 use time::OffsetDateTime;
 
 use super::error::Result;
@@ -39,7 +39,7 @@ impl Drop for HeedAuthStore {
     }
 }
 
-pub fn open_auth_store_env(path: &Path) -> heed::Result<heed::Env> {
+pub fn open_auth_store_env(path: &Path) -> milli::heed::Result<milli::heed::Env> {
     let mut options = EnvOpenOptions::new();
     options.map_size(AUTH_STORE_SIZE); // 1GB
     options.max_dbs(2);
@@ -203,7 +203,7 @@ impl HeedAuthStore {
 /// optionnally on a spcific index, for a given key.
 pub struct KeyIdActionCodec;
 
-impl<'a> heed::BytesDecode<'a> for KeyIdActionCodec {
+impl<'a> milli::heed::BytesDecode<'a> for KeyIdActionCodec {
     type DItem = (KeyId, Action, Option<&'a [u8]>);
 
     fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
@@ -218,7 +218,7 @@ impl<'a> heed::BytesDecode<'a> for KeyIdActionCodec {
     }
 }
 
-impl<'a> heed::BytesEncode<'a> for KeyIdActionCodec {
+impl<'a> milli::heed::BytesEncode<'a> for KeyIdActionCodec {
     type EItem = (&'a KeyId, &'a Action, Option<&'a [u8]>);
 
     fn bytes_encode((key_id, action, index): &Self::EItem) -> Option<Cow<[u8]>> {

meilisearch-http/Cargo.toml

@@ -39,7 +39,6 @@ flate2 = "1.0.21"
 fst = "0.4.7"
 futures = "0.3.17"
 futures-util = "0.3.17"
-heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
 http = "0.2.4"
 indexmap = { version = "1.7.0", features = ["serde-1"] }
 iso8601-duration = "0.1.0"
@@ -77,6 +76,7 @@ tokio = { version = "1.11.0", features = ["full"] }
 tokio-stream = "0.1.7"
 uuid = { version = "0.8.2", features = ["serde"] }
 walkdir = "2.3.2"
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.24.0" }
 pin-project-lite = "0.2.8"
 
 [dev-dependencies]

meilisearch-http/src/helpers/env.rs

@@ -4,7 +4,7 @@ pub trait EnvSizer {
     fn size(&self) -> u64;
 }
 
-impl EnvSizer for heed::Env {
+impl EnvSizer for milli::heed::Env {
     fn size(&self) -> u64 {
         WalkDir::new(self.path())
             .into_iter()

meilisearch-lib/Cargo.toml

@@ -19,7 +19,6 @@ flate2 = "1.0.21"
 fst = "0.4.7"
 futures = "0.3.17"
 futures-util = "0.3.17"
-heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
 http = "0.2.4"
 indexmap = { version = "1.7.0", features = ["serde-1"] }
 itertools = "0.10.1"
@@ -27,7 +26,7 @@ lazy_static = "1.4.0"
 log = "0.4.14"
 meilisearch-error = { path = "../meilisearch-error" }
 meilisearch-auth = { path = "../meilisearch-auth" }
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.22.2" }
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.24.0" }
 mime = "0.3.16"
 num_cpus = "1.13.0"
 once_cell = "1.8.0"

meilisearch-lib/src/index/dump.rs

@@ -3,9 +3,9 @@ use std::io::{BufReader, Seek, SeekFrom, Write};
 use std::path::Path;
 
 use anyhow::Context;
-use heed::{EnvOpenOptions, RoTxn};
 use indexmap::IndexMap;
 use milli::documents::DocumentBatchReader;
+use milli::heed::{EnvOpenOptions, RoTxn};
 use milli::update::{IndexDocumentsConfig, IndexerConfig};
 use serde::{Deserialize, Serialize};
 

meilisearch-lib/src/index/error.rs

@@ -21,7 +21,7 @@ pub enum IndexError {
 
 internal_error!(
     IndexError: std::io::Error,
-    heed::Error,
+    milli::heed::Error,
     fst::Error,
     serde_json::Error,
     update_file_store::UpdateFileStoreError,

meilisearch-lib/src/index/index.rs

@@ -5,7 +5,7 @@ use std::ops::Deref;
 use std::path::Path;
 use std::sync::Arc;
 
-use heed::{EnvOpenOptions, RoTxn};
+use milli::heed::{EnvOpenOptions, RoTxn};
 use milli::update::{IndexerConfig, Setting};
 use milli::{obkv_to_json, FieldDistribution, FieldId};
 use serde::{Deserialize, Serialize};
@@ -37,7 +37,7 @@ impl IndexMeta {
         Self::new_txn(index, &txn)
     }
 
-    pub fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result<Self> {
+    pub fn new_txn(index: &Index, txn: &milli::heed::RoTxn) -> Result<Self> {
         let created_at = index.created_at(txn)?;
         let updated_at = index.updated_at(txn)?;
         let primary_key = index.primary_key(txn)?.map(String::from);
@@ -250,7 +250,7 @@ impl Index {
 
     fn fields_to_display<S: AsRef<str>>(
         &self,
-        txn: &heed::RoTxn,
+        txn: &milli::heed::RoTxn,
         attributes_to_retrieve: &Option<Vec<S>>,
         fields_ids_map: &milli::FieldsIdsMap,
     ) -> Result<Vec<FieldId>> {
@@ -278,7 +278,7 @@ impl Index {
         let _txn = self.write_txn()?;
         self.inner
             .env
-            .copy_to_path(dst, heed::CompactionOption::Enabled)?;
+            .copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
         Ok(())
     }
 }

meilisearch-lib/src/index/updates.rs

@@ -176,7 +176,7 @@ pub struct Facets {
 impl Index {
     fn update_primary_key_txn<'a, 'b>(
         &'a self,
-        txn: &mut heed::RwTxn<'a, 'b>,
+        txn: &mut milli::heed::RwTxn<'a, 'b>,
         primary_key: String,
     ) -> Result<IndexMeta> {
         let mut builder = milli::update::Settings::new(txn, self, self.indexer_config.as_ref());

meilisearch-lib/src/index_controller/dump_actor/error.rs

@@ -18,7 +18,7 @@ pub enum DumpActorError {
 }
 
 internal_error!(
-    DumpActorError: heed::Error,
+    DumpActorError: milli::heed::Error,
     std::io::Error,
     tokio::task::JoinError,
     tokio::sync::oneshot::error::RecvError,

meilisearch-lib/src/index_controller/dump_actor/loaders/v4.rs

@@ -1,9 +1,9 @@
 use std::path::Path;
 use std::sync::Arc;
 
-use heed::EnvOpenOptions;
 use log::info;
 use meilisearch_auth::AuthController;
+use milli::heed::EnvOpenOptions;
 
 use crate::analytics;
 use crate::index_controller::dump_actor::Metadata;

meilisearch-lib/src/index_controller/mod.rs

@@ -48,8 +48,8 @@ pub type Payload = Box<
     dyn Stream<Item = std::result::Result<Bytes, PayloadError>> + Send + Sync + 'static + Unpin,
 >;
 
-pub fn open_meta_env(path: &Path, size: usize) -> heed::Result<heed::Env> {
+pub fn open_meta_env(path: &Path, size: usize) -> milli::heed::Result<milli::heed::Env> {
-    let mut options = heed::EnvOpenOptions::new();
+    let mut options = milli::heed::EnvOpenOptions::new();
     options.map_size(size);
     options.max_dbs(20);
     options.open(path)

meilisearch-lib/src/index_resolver/error.rs

@@ -45,7 +45,7 @@ impl From<OneshotRecvError> for IndexResolverError {
 }
 
 internal_error!(
-    IndexResolverError: heed::Error,
+    IndexResolverError: milli::heed::Error,
     uuid::Error,
     std::io::Error,
     tokio::task::JoinError,

meilisearch-lib/src/index_resolver/meta_store.rs

@@ -4,8 +4,8 @@ use std::io::{BufRead, BufReader, Write};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use heed::types::{SerdeBincode, Str};
+use milli::heed::types::{SerdeBincode, Str};
-use heed::{CompactionOption, Database, Env};
+use milli::heed::{CompactionOption, Database, Env};
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
@@ -56,7 +56,7 @@ impl Drop for HeedMetaStore {
 }
 
 impl HeedMetaStore {
-    pub fn new(env: Arc<heed::Env>) -> Result<Self> {
+    pub fn new(env: Arc<milli::heed::Env>) -> Result<Self> {
         let db = env.create_database(Some("uuids"))?;
         Ok(Self { env, db })
     }
@@ -153,7 +153,7 @@ impl HeedMetaStore {
         Ok(())
     }
 
-    pub fn load_dump(src: impl AsRef<Path>, env: Arc<heed::Env>) -> Result<()> {
+    pub fn load_dump(src: impl AsRef<Path>, env: Arc<milli::heed::Env>) -> Result<()> {
         let src_indexes = src.as_ref().join(UUIDS_DB_PATH).join("data.jsonl");
         let indexes = File::open(&src_indexes)?;
         let mut indexes = BufReader::new(indexes);

meilisearch-lib/src/index_resolver/mod.rs

@@ -7,10 +7,10 @@ use std::path::Path;
 use std::sync::Arc;
 
 use error::{IndexResolverError, Result};
-use heed::Env;
 use index_store::{IndexStore, MapIndexStore};
 use meilisearch_error::ResponseError;
 use meta_store::{HeedMetaStore, IndexMetaStore};
+use milli::heed::Env;
 use milli::update::{DocumentDeletionResult, IndexerConfig};
 use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
@@ -39,7 +39,7 @@ pub fn create_index_resolver(
     path: impl AsRef<Path>,
     index_size: usize,
     indexer_opts: &IndexerOpts,
-    meta_env: Arc<heed::Env>,
+    meta_env: Arc<milli::heed::Env>,
     file_store: UpdateFileStore,
 ) -> anyhow::Result<HardStateIndexResolver> {
     let uuid_store = HeedMetaStore::new(meta_env)?;

meilisearch-lib/src/lib.rs

@@ -25,7 +25,7 @@ pub trait EnvSizer {
     fn size(&self) -> u64;
 }
 
-impl EnvSizer for heed::Env {
+impl EnvSizer for milli::heed::Env {
     fn size(&self) -> u64 {
         WalkDir::new(self.path())
             .into_iter()

meilisearch-lib/src/snapshot.rs

@@ -149,7 +149,7 @@ impl SnapshotJob {
         let env = open_meta_env(&self.src_path, self.meta_env_size)?;
 
         let dst = path.join("data.mdb");
-        env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
+        env.copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
 
         Ok(())
     }
@@ -180,12 +180,12 @@ impl SnapshotJob {
 
             let dst = dst.join("data.mdb");
 
-            let mut options = heed::EnvOpenOptions::new();
+            let mut options = milli::heed::EnvOpenOptions::new();
             options.map_size(self.index_size);
             let index = milli::Index::new(options, entry.path())?;
             index
                 .env
-                .copy_to_path(dst, heed::CompactionOption::Enabled)?;
+                .copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
         }
 
         Ok(())
@@ -198,7 +198,7 @@ impl SnapshotJob {
         let dst = dst.join("data.mdb");
 
         let env = open_auth_store_env(&auth_path)?;
-        env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
+        env.copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
 
         Ok(())
     }

meilisearch-lib/src/tasks/error.rs

@@ -16,7 +16,7 @@ pub enum TaskError {
 }
 
 internal_error!(
-    TaskError: heed::Error,
+    TaskError: milli::heed::Error,
     JoinError,
     std::io::Error,
     serde_json::Error,

meilisearch-lib/src/tasks/task_store/mod.rs

@@ -5,8 +5,8 @@ use std::io::{BufWriter, Write};
 use std::path::Path;
 use std::sync::Arc;
 
-use heed::{Env, RwTxn};
 use log::debug;
+use milli::heed::{Env, RwTxn};
 use time::OffsetDateTime;
 
 use super::error::TaskError;
@@ -61,7 +61,7 @@ impl Clone for TaskStore {
 }
 
 impl TaskStore {
-    pub fn new(env: Arc<heed::Env>) -> Result<Self> {
+    pub fn new(env: Arc<milli::heed::Env>) -> Result<Self> {
         let store = Arc::new(Store::new(env)?);
         Ok(Self { store })
     }
@@ -248,7 +248,7 @@ pub mod test {
     }
 
     impl MockTaskStore {
-        pub fn new(env: Arc<heed::Env>) -> Result<Self> {
+        pub fn new(env: Arc<milli::heed::Env>) -> Result<Self> {
             Ok(Self::Real(TaskStore::new(env)?))
         }
 

meilisearch-lib/src/tasks/task_store/store.rs

@@ -1,5 +1,5 @@
 #[allow(clippy::upper_case_acronyms)]
-type BEU64 = heed::zerocopy::U64<heed::byteorder::BE>;
+type BEU64 = milli::heed::zerocopy::U64<milli::heed::byteorder::BE>;
 
 const UID_TASK_IDS: &str = "uid_task_id";
 const TASKS: &str = "tasks";
@@ -12,8 +12,8 @@ use std::ops::Range;
 use std::result::Result as StdResult;
 use std::sync::Arc;
 
-use heed::types::{ByteSlice, OwnedType, SerdeJson, Unit};
+use milli::heed::types::{ByteSlice, OwnedType, SerdeJson, Unit};
-use heed::{BytesDecode, BytesEncode, Database, Env, RoTxn, RwTxn};
+use milli::heed::{BytesDecode, BytesEncode, Database, Env, RoTxn, RwTxn};
 
 use crate::tasks::task::{Task, TaskId};
 
@@ -73,7 +73,7 @@ impl Store {
     /// be in an invalid state, with dangling processing tasks.
     /// You want to patch  all un-finished tasks and put them in your pending
     /// queue with the `reset_and_return_unfinished_update` method.
-    pub fn new(env: Arc<heed::Env>) -> Result<Self> {
+    pub fn new(env: Arc<milli::heed::Env>) -> Result<Self> {
         let uids_task_ids = env.create_database(Some(UID_TASK_IDS))?;
         let tasks = env.create_database(Some(TASKS))?;
 
@@ -130,7 +130,7 @@ impl Store {
         let range = from..limit
             .map(|limit| (limit as u64).saturating_add(from))
             .unwrap_or(u64::MAX);
-        let iter: Box<dyn Iterator<Item = StdResult<_, heed::Error>>> = match filter {
+        let iter: Box<dyn Iterator<Item = StdResult<_, milli::heed::Error>>> = match filter {
             Some(
                 ref filter @ TaskFilter {
                     indexes: Some(_), ..
@@ -150,7 +150,7 @@ impl Store {
             ),
         };
 
-        let apply_fitler = |task: &StdResult<_, heed::Error>| match task {
+        let apply_fitler = |task: &StdResult<_, milli::heed::Error>| match task {
             Ok(ref t) => filter
                 .as_ref()
                 .and_then(|filter| filter.filter_fn.as_ref())
@@ -162,7 +162,7 @@ impl Store {
         let tasks = iter
             .filter(apply_fitler)
             .take(limit.unwrap_or(usize::MAX))
-            .try_fold::<_, _, StdResult<_, heed::Error>>(Vec::new(), |mut v, task| {
+            .try_fold::<_, _, StdResult<_, milli::heed::Error>>(Vec::new(), |mut v, task| {
                 v.push(task?);
                 Ok(v)
             })?;
@@ -172,7 +172,7 @@ impl Store {
 
     fn compute_candidates(
         &self,
-        txn: &heed::RoTxn,
+        txn: &milli::heed::RoTxn,
         filter: &TaskFilter,
         range: Range<TaskId>,
     ) -> Result<BinaryHeap<TaskId>> {
@@ -188,10 +188,10 @@ impl Store {
                 self.uids_task_ids
                     .remap_key_type::<ByteSlice>()
                     .rev_prefix_iter(txn, &index_uid)?
-                    .map(|entry| -> StdResult<_, heed::Error> {
+                    .map(|entry| -> StdResult<_, milli::heed::Error> {
                         let (key, _) = entry?;
-                        let (_, id) =
+                        let (_, id) = IndexUidTaskIdCodec::bytes_decode(key)
-                            IndexUidTaskIdCodec::bytes_decode(key).ok_or(heed::Error::Decoding)?;
+                            .ok_or(milli::heed::Error::Decoding)?;
                         Ok(id)
                     })
                     .skip_while(|entry| {
@@ -212,7 +212,7 @@ impl Store {
                             // if we encounter an error we returns true to collect it later
                             .unwrap_or(true)
                     })
-                    .try_for_each::<_, StdResult<(), heed::Error>>(|id| {
+                    .try_for_each::<_, StdResult<(), milli::heed::Error>>(|id| {
                         candidates.push(id?);
                         Ok(())
                     })?;
@@ -225,8 +225,8 @@ impl Store {
 
 #[cfg(test)]
 pub mod test {
-    use heed::EnvOpenOptions;
     use itertools::Itertools;
+    use milli::heed::EnvOpenOptions;
     use nelson::Mocker;
     use proptest::collection::vec;
     use proptest::prelude::*;
@@ -244,10 +244,10 @@ pub mod test {
         Fake(Mocker),
     }
 
-    pub struct TmpEnv(TempDir, Arc<heed::Env>);
+    pub struct TmpEnv(TempDir, Arc<milli::heed::Env>);
 
     impl TmpEnv {
-        pub fn env(&self) -> Arc<heed::Env> {
+        pub fn env(&self) -> Arc<milli::heed::Env> {
             self.1.clone()
         }
     }
@@ -264,7 +264,7 @@ pub mod test {
     }
 
     impl MockStore {
-        pub fn new(env: Arc<heed::Env>) -> Result<Self> {
+        pub fn new(env: Arc<milli::heed::Env>) -> Result<Self> {
             Ok(Self::Real(Store::new(env)?))
         }