From deee22b5da1226ebe4cd98aa922bf476fb056b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 3 Jul 2024 17:51:53 +0200 Subject: [PATCH] Use an experimental feature to avoid copying 64k in memory --- Cargo.lock | 155 ++--------------------------------------- meilisearch/Cargo.toml | 2 +- milli/Cargo.toml | 2 +- milli/src/index.rs | 2 +- 4 files changed, 9 insertions(+), 152 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53b46271d..7ffbc5d5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,17 +234,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.7.8" @@ -728,27 +717,6 @@ dependencies = [ "bytes", ] -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "camino" version = "1.1.6" @@ -967,16 +935,6 @@ dependencies = [ "half 1.8.2", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clang-sys" version = "1.7.0" @@ -1070,12 +1028,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "constant_time_eq" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" - [[package]] name = "convert_case" version = "0.4.0" @@ -1123,21 +1075,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.4.2" @@ -1365,12 +1302,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "deflate64" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" - [[package]] name = "deranged" version = "0.3.11" @@ -2548,15 +2479,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] - [[package]] name = "insta" version = "1.39.0" @@ -3259,16 +3181,6 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" -[[package]] -name = "lzma-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" -dependencies = [ - "byteorder", - "crc", -] - [[package]] name = "macro_rules_attribute" version = "0.2.0" @@ -3544,7 +3456,7 @@ dependencies = [ "ureq", "url", "uuid", - "zstd 0.11.2+zstd.1.5.2", + "zstd", ] [[package]] @@ -3889,16 +3801,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498a099351efa4becc6a19c72aa9270598e8fd274ca47052e37455241c88b696" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest", - "hmac", -] - [[package]] name = "pem" version = "3.0.3" @@ -6383,20 +6285,6 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] [[package]] name = "zerovec" @@ -6428,27 +6316,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" dependencies = [ - "aes", "arbitrary", - "bzip2", - "constant_time_eq", "crc32fast", "crossbeam-utils", - "deflate64", "displaydoc", "flate2", - "hmac", "indexmap", - "lzma-rs", "memchr", - "pbkdf2", - "rand", - "sha1", "thiserror", - "time", - "zeroize", "zopfli", - "zstd 0.13.2", ] [[package]] @@ -6467,37 +6343,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.11.2+zstd.1.5.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", -] - -[[package]] -name = "zstd" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" -dependencies = [ - "zstd-safe 7.2.0", + "zstd-safe", ] [[package]] name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-safe" -version = "7.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" dependencies = [ "zstd-sys", ] diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index 9119651db..7ecdf915e 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -125,7 +125,7 @@ reqwest = { version = "0.12.5", features = [ sha-1 = { version = "0.10.1", optional = true } static-files = { version = "0.2.4", optional = true } tempfile = { version = "3.10.1", optional = true } -zip = { version = "2.1.3", optional = true } +zip = { version = "2.1.3", default-features = false, features = ["deflate"], optional = true } [features] default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"] diff --git a/milli/Cargo.toml b/milli/Cargo.toml index 3220ec217..4bd74e865 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -38,7 +38,7 @@ heed = { version = "0.20.3", default-features = false, features = [ indexmap = { version = "2.2.6", features = ["serde"] } json-depth-checker = { path = "../json-depth-checker" } levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] } -zstd = { version = "0.11.2", features = ["zdict_builder"] } +zstd = { version = "0.13.1", features = ["zdict_builder", "experimental"] } memmap2 = "0.9.4" obkv = "0.2.2" once_cell = "1.19.0" diff --git a/milli/src/index.rs b/milli/src/index.rs index 8db7c89cc..db39d61bf 100644 --- a/milli/src/index.rs +++ b/milli/src/index.rs @@ -374,7 +374,7 @@ impl Index { &self, rtxn: &'t RoTxn, ) -> heed::Result>> { - self.document_compression_raw_dictionary(rtxn).map(|opt| opt.map(DecoderDictionary::copy)) + self.document_compression_raw_dictionary(rtxn).map(|opt| opt.map(DecoderDictionary::new)) } pub fn document_compression_dictionary(