From bdc5599b73d44984e8012763ca4a1e1f64d39cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Mon, 28 Jun 2021 18:26:20 +0200 Subject: [PATCH] Bump heed to use the git repo with v0.12.0 --- Cargo.lock | 65 +++++++++++++++++++++---- helpers/Cargo.toml | 2 +- http-ui/Cargo.toml | 2 +- infos/Cargo.toml | 2 +- milli/Cargo.toml | 2 +- milli/src/update/delete_documents.rs | 57 ++++++++++++++-------- milli/src/update/index_documents/mod.rs | 12 +++-- search/Cargo.toml | 2 +- 8 files changed, 106 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73114c7c1..70c21b542 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,7 +131,7 @@ dependencies = [ "convert_case", "criterion", "flate2", - "heed", + "heed 0.10.6", "jemallocator", "milli", "reqwest", @@ -868,10 +868,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afcc6c911acaadad3ebe9f1ef1707d80bd71c92037566f47b6238a03b60adf1a" dependencies = [ "byteorder", - "heed-traits", - "heed-types", + "heed-traits 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "heed-types 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc", - "lmdb-rkv-sys", + "lmdb-rkv-sys 0.11.0", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", + "zerocopy", +] + +[[package]] +name = "heed" +version = "0.12.0" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" +dependencies = [ + "byteorder", + "heed-traits 0.7.0 (git+https://github.com/Kerollmops/heed?tag=v0.12.0)", + "heed-types 0.7.2 (git+https://github.com/Kerollmops/heed?tag=v0.12.0)", + "libc", + "lmdb-rkv-sys 0.15.0", "once_cell", "page_size", "serde", @@ -886,6 +904,11 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" +[[package]] +name = "heed-traits" +version = "0.7.0" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" + [[package]] name = "heed-types" version = "0.7.2" @@ -893,7 +916,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" dependencies = [ "bincode", - "heed-traits", + "heed-traits 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde", + "serde_json", + "zerocopy", +] + +[[package]] +name = "heed-types" +version = "0.7.2" +source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326" +dependencies = [ + "bincode", + "heed-traits 0.7.0 (git+https://github.com/Kerollmops/heed?tag=v0.12.0)", "serde", "serde_json", "zerocopy", @@ -905,7 +940,7 @@ version = "0.6.0" dependencies = [ "anyhow", "byte-unit", - "heed", + "heed 0.10.6", "jemallocator", "milli", "stderrlog", @@ -969,7 +1004,7 @@ dependencies = [ "funty", "futures", "grenad", - "heed", + "heed 0.12.0", "jemallocator", "log", "maplit", @@ -1103,7 +1138,7 @@ dependencies = [ "anyhow", "byte-unit", "csv", - "heed", + "heed 0.10.6", "jemallocator", "milli", "roaring", @@ -1275,6 +1310,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "lmdb-rkv-sys" +version = "0.15.0" +source = "git+https://github.com/meilisearch/lmdb-rs#d0b50d02938ee84e4e4372697ea991fe2a4cae3b" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "log" version = "0.4.14" @@ -1389,7 +1434,7 @@ dependencies = [ "fst", "fxhash", "grenad", - "heed", + "heed 0.12.0", "human_format", "itertools 0.10.0", "levenshtein_automata", @@ -2236,7 +2281,7 @@ version = "0.6.0" dependencies = [ "anyhow", "byte-unit", - "heed", + "heed 0.10.6", "jemallocator", "log", "milli", diff --git a/helpers/Cargo.toml b/helpers/Cargo.toml index 224b0e5eb..bd509904d 100644 --- a/helpers/Cargo.toml +++ b/helpers/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] anyhow = "1.0.38" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } -heed = "0.10.6" +heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" } milli = { path = "../milli" } stderrlog = "0.5.1" structopt = { version = "0.3.21", default-features = false } diff --git a/http-ui/Cargo.toml b/http-ui/Cargo.toml index 5e6ab9d09..e9525dd98 100644 --- a/http-ui/Cargo.toml +++ b/http-ui/Cargo.toml @@ -10,7 +10,7 @@ anyhow = "1.0.38" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } crossbeam-channel = "0.5.0" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = "0.10.6" +heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" milli = { path = "../milli" } diff --git a/infos/Cargo.toml b/infos/Cargo.toml index d12036a07..3c7bef2b8 100644 --- a/infos/Cargo.toml +++ b/infos/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" anyhow = "1.0.38" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } csv = "1.1.5" -heed = "0.10.6" +heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" } milli = { path = "../milli" } roaring = "0.6.6" serde_json = "1.0.62" diff --git a/milli/Cargo.toml b/milli/Cargo.toml index 872c339ff..1e6c2a9a0 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -14,7 +14,7 @@ flate2 = "1.0.20" fst = "0.4.5" fxhash = "0.2.1" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] } +heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0", default-features = false, features = ["lmdb", "sync-read-txn"] } human_format = "1.0.3" levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] } linked-hash-map = "0.5.4" diff --git a/milli/src/update/delete_documents.rs b/milli/src/update/delete_documents.rs index 30ae55e62..a0c1f48f5 100644 --- a/milli/src/update/delete_documents.rs +++ b/milli/src/update/delete_documents.rs @@ -132,7 +132,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { }; external_ids.push(external_id); } - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; } drop(iter); @@ -143,7 +144,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { let ((_docid, word), _positions) = result?; // This boolean will indicate if we must remove this word from the words FST. words.push((SmallString32::from(word), false)); - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; } } @@ -194,11 +196,13 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { let previous_len = docids.len(); docids.difference_with(&self.documents_ids); if docids.is_empty() { - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; *must_remove = true; } else if docids.len() != previous_len { let key = key.to_owned(); - iter.put_current(&key, &docids)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(&key, &docids)? }; } } } @@ -243,10 +247,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { let previous_len = docids.len(); docids.difference_with(&self.documents_ids); if docids.is_empty() { - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; prefixes_to_delete.insert(prefix)?; } else if docids.len() != previous_len { - iter.put_current(&prefix, &docids)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(&prefix, &docids)? }; } } @@ -281,10 +287,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { let previous_len = docids.len(); docids.difference_with(&self.documents_ids); if docids.is_empty() { - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; } else if docids.len() != previous_len { let key = key.to_owned(); - iter.put_current(&key, &docids)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(&key, &docids)? }; } } @@ -300,10 +308,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { let previous_len = docids.len(); docids.difference_with(&self.documents_ids); if docids.is_empty() { - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; } else if docids.len() != previous_len { let bytes = bytes.to_owned(); - iter.put_current(&bytes, &docids)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(&bytes, &docids)? }; } } @@ -317,10 +327,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { let previous_len = docids.len(); docids.difference_with(&self.documents_ids); if docids.is_empty() { - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; } else if docids.len() != previous_len { let bytes = bytes.to_owned(); - iter.put_current(&bytes, &docids)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(&bytes, &docids)? }; } } @@ -334,10 +346,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { let previous_len = docids.len(); docids.difference_with(&self.documents_ids); if docids.is_empty() { - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; } else if docids.len() != previous_len { let bytes = bytes.to_owned(); - iter.put_current(&bytes, &docids)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(&bytes, &docids)? }; } } @@ -349,9 +363,11 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { let previous_len = docids.len(); docids.difference_with(&self.documents_ids); if docids.is_empty() { - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; } else if docids.len() != previous_len { - iter.put_current(&key, &docids)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(&key, &docids)? }; } } @@ -420,7 +436,8 @@ where while let Some(result) = iter.next() { let (key, ()) = result?; if to_remove.contains(convert(key)) { - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; } } @@ -441,10 +458,12 @@ where let previous_len = docids.len(); docids.difference_with(to_remove); if docids.is_empty() { - iter.del_current()?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.del_current()? }; } else if docids.len() != previous_len { let bytes = bytes.to_owned(); - iter.put_current(&bytes, &docids)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(&bytes, &docids)? }; } } diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 316b0eb81..7faa27588 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -144,7 +144,8 @@ where WriteMethod::Append => { let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?; while let Some((k, v)) = reader.next()? { - out_iter.append(k, v)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { out_iter.append(k, v)? }; } } WriteMethod::GetMergePut => { @@ -154,7 +155,8 @@ where Some((key, old_val)) if key == k => { let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..]; let val = merge(k, &vals)?; - iter.put_current(k, &val)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(k, &val)? }; } _ => { drop(iter); @@ -203,7 +205,8 @@ where WriteMethod::Append => { let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?; while let Some((k, v)) = sorter.next()? { - out_iter.append(k, v)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { out_iter.append(k, v)? }; } } WriteMethod::GetMergePut => { @@ -216,7 +219,8 @@ where // TODO just wrap this error? InternalError::IndexingMergingKeys { process: "get-put-merge" } })?; - iter.put_current(k, &val)?; + // safety: we don't keep references from inside the LMDB database. + unsafe { iter.put_current(k, &val)? }; } _ => { drop(iter); diff --git a/search/Cargo.toml b/search/Cargo.toml index bd1aba2dd..16c82a93f 100644 --- a/search/Cargo.toml +++ b/search/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] anyhow = "1.0.38" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } -heed = "0.10.6" +heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" } log = "0.4.14" milli = { path = "../milli" } serde_json = "1.0.62"