Bump heed to use the git repo with v0.12.0

Cargo.lock

@@ -131,7 +131,7 @@ dependencies = [
  "convert_case",
  "criterion",
  "flate2",
- "heed",
+ "heed 0.10.6",
  "jemallocator",
  "milli",
  "reqwest",
@@ -868,10 +868,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a"
 dependencies = [
  "byteorder",
- "heed-traits",
+ "heed-traits 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "heed-types",
+ "heed-types 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc",
- "lmdb-rkv-sys",
+ "lmdb-rkv-sys 0.11.0",
+ "once_cell",
+ "page_size",
+ "serde",
+ "synchronoise",
+ "url",
+ "zerocopy",
+]
+
+[[package]]
+name = "heed"
+version = "0.12.0"
+source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
+dependencies = [
+ "byteorder",
+ "heed-traits 0.7.0 (git+https://github.com/Kerollmops/heed?tag=v0.12.0)",
+ "heed-types 0.7.2 (git+https://github.com/Kerollmops/heed?tag=v0.12.0)",
+ "libc",
+ "lmdb-rkv-sys 0.15.0",
  "once_cell",
  "page_size",
  "serde",
@@ -886,6 +904,11 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c"
 
+[[package]]
+name = "heed-traits"
+version = "0.7.0"
+source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
+
 [[package]]
 name = "heed-types"
 version = "0.7.2"
@@ -893,7 +916,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405"
 dependencies = [
  "bincode",
- "heed-traits",
+ "heed-traits 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde",
+ "serde_json",
+ "zerocopy",
+]
+
+[[package]]
+name = "heed-types"
+version = "0.7.2"
+source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
+dependencies = [
+ "bincode",
+ "heed-traits 0.7.0 (git+https://github.com/Kerollmops/heed?tag=v0.12.0)",
  "serde",
  "serde_json",
  "zerocopy",
@@ -905,7 +940,7 @@ version = "0.6.0"
 dependencies = [
  "anyhow",
  "byte-unit",
- "heed",
+ "heed 0.10.6",
  "jemallocator",
  "milli",
  "stderrlog",
@@ -969,7 +1004,7 @@ dependencies = [
  "funty",
  "futures",
  "grenad",
- "heed",
+ "heed 0.12.0",
  "jemallocator",
  "log",
  "maplit",
@@ -1103,7 +1138,7 @@ dependencies = [
  "anyhow",
  "byte-unit",
  "csv",
- "heed",
+ "heed 0.10.6",
  "jemallocator",
  "milli",
  "roaring",
@@ -1275,6 +1310,16 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "lmdb-rkv-sys"
+version = "0.15.0"
+source = "git+https://github.com/meilisearch/lmdb-rs#d0b50d02938ee84e4e4372697ea991fe2a4cae3b"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "log"
 version = "0.4.14"
@@ -1389,7 +1434,7 @@ dependencies = [
  "fst",
  "fxhash",
  "grenad",
- "heed",
+ "heed 0.12.0",
  "human_format",
  "itertools 0.10.0",
  "levenshtein_automata",
@@ -2236,7 +2281,7 @@ version = "0.6.0"
 dependencies = [
  "anyhow",
  "byte-unit",
- "heed",
+ "heed 0.10.6",
  "jemallocator",
  "log",
  "milli",

helpers/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0.38"
 byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
-heed = "0.10.6"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
 milli = { path = "../milli" }
 stderrlog = "0.5.1"
 structopt = { version = "0.3.21", default-features = false }

http-ui/Cargo.toml

@@ -10,7 +10,7 @@ anyhow = "1.0.38"
 byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
 crossbeam-channel = "0.5.0"
 grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
-heed = "0.10.6"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
 meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" }
 memmap = "0.7.0"
 milli = { path = "../milli" }

infos/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2018"
 anyhow = "1.0.38"
 byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
 csv = "1.1.5"
-heed = "0.10.6"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
 milli = { path = "../milli" }
 roaring = "0.6.6"
 serde_json = "1.0.62"

milli/Cargo.toml

@@ -14,7 +14,7 @@ flate2 = "1.0.20"
 fst = "0.4.5"
 fxhash = "0.2.1"
 grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
-heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] }
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0", default-features = false, features = ["lmdb", "sync-read-txn"] }
 human_format = "1.0.3"
 levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
 linked-hash-map = "0.5.4"

milli/src/update/delete_documents.rs

@@ -132,7 +132,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
                     };
                     external_ids.push(external_id);
                 }
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             }
             drop(iter);
 
@@ -143,7 +144,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
                 let ((_docid, word), _positions) = result?;
                 // This boolean will indicate if we must remove this word from the words FST.
                 words.push((SmallString32::from(word), false));
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             }
         }
 
@@ -194,11 +196,13 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
                     let previous_len = docids.len();
                     docids.difference_with(&self.documents_ids);
                     if docids.is_empty() {
-                        iter.del_current()?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.del_current()? };
                         *must_remove = true;
                     } else if docids.len() != previous_len {
                         let key = key.to_owned();
-                        iter.put_current(&key, &docids)?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.put_current(&key, &docids)? };
                     }
                 }
             }
@@ -243,10 +247,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
                 prefixes_to_delete.insert(prefix)?;
             } else if docids.len() != previous_len {
-                iter.put_current(&prefix, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&prefix, &docids)? };
             }
         }
 
@@ -281,10 +287,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let key = key.to_owned();
-                iter.put_current(&key, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&key, &docids)? };
             }
         }
 
@@ -300,10 +308,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let bytes = bytes.to_owned();
-                iter.put_current(&bytes, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&bytes, &docids)? };
             }
         }
 
@@ -317,10 +327,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let bytes = bytes.to_owned();
-                iter.put_current(&bytes, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&bytes, &docids)? };
             }
         }
 
@@ -334,10 +346,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let bytes = bytes.to_owned();
-                iter.put_current(&bytes, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&bytes, &docids)? };
             }
         }
 
@@ -349,9 +363,11 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
-                iter.put_current(&key, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&key, &docids)? };
             }
         }
 
@@ -420,7 +436,8 @@ where
     while let Some(result) = iter.next() {
         let (key, ()) = result?;
         if to_remove.contains(convert(key)) {
-            iter.del_current()?;
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.del_current()? };
         }
     }
 
@@ -441,10 +458,12 @@ where
         let previous_len = docids.len();
         docids.difference_with(to_remove);
         if docids.is_empty() {
-            iter.del_current()?;
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.del_current()? };
         } else if docids.len() != previous_len {
             let bytes = bytes.to_owned();
-            iter.put_current(&bytes, &docids)?;
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.put_current(&bytes, &docids)? };
         }
     }
 

milli/src/update/index_documents/mod.rs

@@ -144,7 +144,8 @@ where
         WriteMethod::Append => {
             let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
             while let Some((k, v)) = reader.next()? {
-                out_iter.append(k, v)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { out_iter.append(k, v)? };
             }
         }
         WriteMethod::GetMergePut => {
@@ -154,7 +155,8 @@ where
                     Some((key, old_val)) if key == k => {
                         let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..];
                         let val = merge(k, &vals)?;
-                        iter.put_current(k, &val)?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.put_current(k, &val)? };
                     }
                     _ => {
                         drop(iter);
@@ -203,7 +205,8 @@ where
         WriteMethod::Append => {
             let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
             while let Some((k, v)) = sorter.next()? {
-                out_iter.append(k, v)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { out_iter.append(k, v)? };
             }
         }
         WriteMethod::GetMergePut => {
@@ -216,7 +219,8 @@ where
                             // TODO just wrap this error?
                             InternalError::IndexingMergingKeys { process: "get-put-merge" }
                         })?;
-                        iter.put_current(k, &val)?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.put_current(k, &val)? };
                     }
                     _ => {
                         drop(iter);

search/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0.38"
 byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
-heed = "0.10.6"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
 log = "0.4.14"
 milli = { path = "../milli" }
 serde_json = "1.0.62"