snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 02:27:40 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Move from oxidized-mtbl to grenad

Browse Source
This commit is contained in:
Kerollmops 2020-10-14 09:59:27 +02:00 committed by Clément Renault
parent b342a86c15
commit f980422c57
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
3 changed files with 130 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
Cargo.lock generated
View File

@ -155,7 +155,7 @@ version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31accafdb70df7871592c058eca3985b71104e15ac32f64706022c58867da931" checksum = "31accafdb70df7871592c058eca3985b71104e15ac32f64706022c58867da931"
dependencies = [ dependencies = [
"lazy_static 1.4.0", "lazy_static",
"memchr", "memchr",
"regex-automata", "regex-automata",
"serde", "serde",
@ -251,12 +251,6 @@ dependencies = [
"bitflags", "bitflags",
] ]
[[package]]
name = "crc32c"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ba37ef26c12988c1cee882d522d65e1d5d2ad8c3864665b88ee92767ed84c5"
[[package]] [[package]]
name = "crc32fast" name = "crc32fast"
version = "1.2.0" version = "1.2.0"
@ -268,9 +262,9 @@ dependencies = [
[[package]] [[package]]
name = "criterion" name = "criterion"
version = "0.3.2" version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63f696897c88b57f4ffe3c69d8e1a0613c7d0e6c4833363c8560fbde9c47b966" checksum = "70daa7ceec6cf143990669a04c7df13391d55fb27bd4079d252fca774ba244d8"
dependencies = [ dependencies = [
"atty", "atty",
"cast", "cast",
@ -278,13 +272,14 @@ dependencies = [
"criterion-plot", "criterion-plot",
"csv", "csv",
"itertools", "itertools",
"lazy_static 1.4.0", "lazy_static",
"num-traits", "num-traits",
"oorandom", "oorandom",
"plotters", "plotters",
"rayon", "rayon",
"regex", "regex",
"serde", "serde",
"serde_cbor",
"serde_derive", "serde_derive",
"serde_json", "serde_json",
"tinytemplate", "tinytemplate",
@ -293,9 +288,9 @@ dependencies = [
[[package]] [[package]]
name = "criterion-plot" name = "criterion-plot"
version = "0.4.2" version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddeaf7989f00f2e1d871a26a110f3ed713632feac17f65f03ca938c542618b60" checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d"
dependencies = [ dependencies = [
"cast", "cast",
"itertools", "itertools",
@ -321,7 +316,7 @@ dependencies = [
"autocfg 1.0.0", "autocfg 1.0.0",
"cfg-if", "cfg-if",
"crossbeam-utils", "crossbeam-utils",
"lazy_static 1.4.0", "lazy_static",
"maybe-uninit", "maybe-uninit",
"memoffset", "memoffset",
"scopeguard", "scopeguard",
@ -345,7 +340,7 @@ checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [ dependencies = [
"autocfg 1.0.0", "autocfg 1.0.0",
"cfg-if", "cfg-if",
"lazy_static 1.4.0", "lazy_static",
] ]
[[package]] [[package]]
@ -579,6 +574,20 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "grenad"
version = "0.1.0"
source = "git+https://github.com/Kerollmops/grenad.git?rev=a884670#a8846703ddee1f1ed86efd3168561606c244e135"
dependencies = [
"byteorder",
"flate2",
"log 0.4.11",
"nix",
"snap",
"tempfile",
"zstd",
]
[[package]] [[package]]
name = "h2" name = "h2"
version = "0.2.5" version = "0.2.5"
@ -598,6 +607,12 @@ dependencies = [
"tokio-util", "tokio-util",
] ]
[[package]]
name = "half"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.8.2" version = "0.8.2"
@ -835,9 +850,9 @@ dependencies = [
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.40" version = "0.3.45"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8"
dependencies = [ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
@ -852,12 +867,6 @@ dependencies = [
"winapi-build", "winapi-build",
] ]
[[package]]
name = "lazy_static"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73"
[[package]] [[package]]
name = "lazy_static" name = "lazy_static"
version = "1.4.0" version = "1.4.0"
@ -875,9 +884,9 @@ dependencies = [
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.70" version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f" checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743"
[[package]] [[package]]
name = "linked-hash-map" name = "linked-hash-map"
@ -932,16 +941,6 @@ version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
[[package]]
name = "memmap"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
dependencies = [
"libc",
"winapi 0.3.8",
]
[[package]] [[package]]
name = "memoffset" name = "memoffset"
version = "0.5.4" version = "0.5.4"
@ -965,6 +964,7 @@ dependencies = [
"flate2", "flate2",
"fst", "fst",
"fxhash", "fxhash",
"grenad",
"heed", "heed",
"human_format", "human_format",
"itertools", "itertools",
@ -972,10 +972,8 @@ dependencies = [
"levenshtein_automata", "levenshtein_automata",
"linked-hash-map", "linked-hash-map",
"log 0.4.11", "log 0.4.11",
"memmap",
"near-proximity", "near-proximity",
"once_cell", "once_cell",
"oxidized-mtbl",
"rayon", "rayon",
"ringtail", "ringtail",
"roaring", "roaring",
@ -1137,6 +1135,18 @@ dependencies = [
"winapi 0.3.8", "winapi 0.3.8",
] ]
[[package]]
name = "nix"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85db2feff6bf70ebc3a4793191517d5f0331100a2f10f9bf93b5e5214f32b7b7"
dependencies = [
"bitflags",
"cc",
"cfg-if",
"libc",
]
[[package]] [[package]]
name = "nom" name = "nom"
version = "5.1.2" version = "5.1.2"
@ -1200,21 +1210,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
[[package]]
name = "oxidized-mtbl"
version = "0.1.0"
source = "git+https://github.com/Kerollmops/oxidized-mtbl.git?rev=9ff8082#9ff808236980305183b2440f08cf471c58e24961"
dependencies = [
"byteorder",
"crc32c",
"flate2",
"log 0.4.11",
"memmap",
"snap",
"tempfile",
"zstd",
]
[[package]] [[package]]
name = "page_size" name = "page_size"
version = "0.4.2" version = "0.4.2"
@ -1556,7 +1551,7 @@ dependencies = [
"crossbeam-deque", "crossbeam-deque",
"crossbeam-queue", "crossbeam-queue",
"crossbeam-utils", "crossbeam-utils",
"lazy_static 1.4.0", "lazy_static",
"num_cpus", "num_cpus",
] ]
@ -1586,9 +1581,9 @@ dependencies = [
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.3.9" version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b"
dependencies = [ dependencies = [
"regex-syntax", "regex-syntax",
] ]
@ -1604,9 +1599,9 @@ dependencies = [
[[package]] [[package]]
name = "regex-syntax" name = "regex-syntax"
version = "0.6.18" version = "0.6.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c"
[[package]] [[package]]
name = "remove_dir_all" name = "remove_dir_all"
@ -1698,6 +1693,16 @@ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]]
name = "serde_cbor"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622"
dependencies = [
"half",
"serde",
]
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.110" version = "1.0.110"
@ -1807,9 +1812,9 @@ dependencies = [
[[package]] [[package]]
name = "stderrlog" name = "stderrlog"
version = "0.4.3" version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e5ee9b90a5452c570a0b0ac1c99ae9498db7e56e33d74366de7f2a7add7f25" checksum = "b02f316286ae558d83acc93dd81eaba096e746987a7961d4a9ae026842bae67f"
dependencies = [ dependencies = [
"atty", "atty",
"chrono", "chrono",
@ -1825,7 +1830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef" checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef"
dependencies = [ dependencies = [
"clap", "clap",
"lazy_static 1.4.0", "lazy_static",
"structopt-derive", "structopt-derive",
] ]
@ -1933,12 +1938,11 @@ dependencies = [
[[package]] [[package]]
name = "thread_local" name = "thread_local"
version = "0.3.4" version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1697c4b57aeeb7a536b647165a2825faddffb1d3bad386d507709bd51a90bb14" checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
dependencies = [ dependencies = [
"lazy_static 0.2.11", "lazy_static",
"unreachable",
] ]
[[package]] [[package]]
@ -1986,7 +1990,7 @@ dependencies = [
"fnv", "fnv",
"futures-core", "futures-core",
"iovec", "iovec",
"lazy_static 1.4.0", "lazy_static",
"libc", "libc",
"memchr", "memchr",
"mio", "mio",
@ -2147,15 +2151,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
[[package]]
name = "unreachable"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
dependencies = [
"void",
]
[[package]] [[package]]
name = "url" name = "url"
version = "2.1.1" version = "2.1.1"
@ -2191,12 +2186,6 @@ version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
[[package]] [[package]]
name = "walkdir" name = "walkdir"
version = "2.3.1" version = "2.3.1"
@ -2252,9 +2241,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]] [[package]]
name = "wasm-bindgen" name = "wasm-bindgen"
version = "0.2.63" version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0" checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"wasm-bindgen-macro", "wasm-bindgen-macro",
@ -2262,12 +2251,12 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-backend" name = "wasm-bindgen-backend"
version = "0.2.63" version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ded84f06e0ed21499f6184df0e0cb3494727b0c5da89534e0fcc55c51d812101" checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68"
dependencies = [ dependencies = [
"bumpalo", "bumpalo",
"lazy_static 1.4.0", "lazy_static",
"log 0.4.11", "log 0.4.11",
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2277,9 +2266,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro" name = "wasm-bindgen-macro"
version = "0.2.63" version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3" checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038"
dependencies = [ dependencies = [
"quote", "quote",
"wasm-bindgen-macro-support", "wasm-bindgen-macro-support",
@ -2287,9 +2276,9 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-macro-support" name = "wasm-bindgen-macro-support"
version = "0.2.63" version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92" checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2300,15 +2289,15 @@ dependencies = [
[[package]] [[package]]
name = "wasm-bindgen-shared" name = "wasm-bindgen-shared"
version = "0.2.63" version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd" checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307"
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.40" version = "0.3.45"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17" checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d"
dependencies = [ dependencies = [
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",

8
Cargo.toml
View File

@ -13,15 +13,14 @@ csv = "1.1.3"
flate2 = "1.0.17" flate2 = "1.0.17"
fst = "0.4.4" fst = "0.4.4"
fxhash = "0.2.1" fxhash = "0.2.1"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "dee0815" }
heed = { version = "0.8.1", default-features = false, features = ["lmdb"] } heed = { version = "0.8.1", default-features = false, features = ["lmdb"] }
human_format = "1.0.3" human_format = "1.0.3"
jemallocator = "0.3.2" jemallocator = "0.3.2"
levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] } levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
linked-hash-map = "0.5.3" linked-hash-map = "0.5.3"
memmap = "0.7.0"
near-proximity = { git = "https://github.com/Kerollmops/plane-sweep-proximity", rev = "6608205" } near-proximity = { git = "https://github.com/Kerollmops/plane-sweep-proximity", rev = "6608205" }
once_cell = "1.4.0" once_cell = "1.4.0"
oxidized-mtbl = { git = "https://github.com/Kerollmops/oxidized-mtbl.git", rev = "9ff8082" }
rayon = "1.3.1" rayon = "1.3.1"
ringtail = "0.3.0" ringtail = "0.3.0"
roaring = "0.6.1" roaring = "0.6.1"
@ -36,7 +35,7 @@ itertools = "0.9.0"
# logging # logging
log = "0.4.11" log = "0.4.11"
stderrlog = "0.4.3" stderrlog = "0.5.0"
# http server # http server
askama = "0.10.1" askama = "0.10.1"
@ -46,13 +45,14 @@ tokio = { version = "0.2.15", features = ["full"] }
warp = "0.2.2" warp = "0.2.2"
[dev-dependencies] [dev-dependencies]
criterion = "0.3" criterion = "0.3.3"
[build-dependencies] [build-dependencies]
fst = "0.4.4" fst = "0.4.4"
[features] [features]
default = [] default = []
file-fuse = ["grenad/file-fuse"]
[[bench]] [[bench]]
name = "search" name = "search"

80
src/bin/indexer.rs
View File

@ -1,7 +1,7 @@
use std::collections::{BTreeMap, HashMap}; use std::collections::{BTreeMap, HashMap};
use std::convert::TryFrom; use std::convert::TryFrom;
use std::fs::File; use std::fs::File;
use std::io::{self, Read, Write}; use std::io::{self, Read, Write, Seek, SeekFrom};
use std::iter::FromIterator; use std::iter::FromIterator;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::mpsc::sync_channel; use std::sync::mpsc::sync_channel;
@ -16,8 +16,7 @@ use fst::IntoStreamer;
use heed::{EnvOpenOptions, BytesEncode, types::ByteSlice}; use heed::{EnvOpenOptions, BytesEncode, types::ByteSlice};
use linked_hash_map::LinkedHashMap; use linked_hash_map::LinkedHashMap;
use log::{debug, info}; use log::{debug, info};
use memmap::Mmap; use grenad::{Reader, FileFuse, Writer, Merger, Sorter, CompressionType};
use oxidized_mtbl::{Reader, Writer, Merger, Sorter, CompressionType};
use rayon::prelude::*; use rayon::prelude::*;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use structopt::StructOpt; use structopt::StructOpt;
@ -110,6 +109,9 @@ struct IndexerOpt {
/// The level of compression of the chosen algorithm. /// The level of compression of the chosen algorithm.
#[structopt(long, requires = "chunk-compression-type")] #[structopt(long, requires = "chunk-compression-type")]
chunk_compression_level: Option<u32>, chunk_compression_level: Option<u32>,
#[structopt(long, default_value = "4294967296")] // 4 GB
file_fusing_shrink_size: u64,
} }
fn format_count(n: usize) -> String { fn format_count(n: usize) -> String {
@ -120,7 +122,7 @@ fn lmdb_key_valid_size(key: &[u8]) -> bool {
!key.is_empty() && key.len() <= LMDB_MAX_KEY_LENGTH !key.is_empty() && key.len() <= LMDB_MAX_KEY_LENGTH
} }
fn create_writer(typ: CompressionType, level: Option<u32>, file: File) -> Writer<File> { fn create_writer(typ: CompressionType, level: Option<u32>, file: File) -> io::Result<Writer<File>> {
let mut builder = Writer::builder(); let mut builder = Writer::builder();
builder.compression_type(typ); builder.compression_type(typ);
if let Some(level) = level { if let Some(level) = level {
@ -129,21 +131,24 @@ fn create_writer(typ: CompressionType, level: Option<u32>, file: File) -> Writer
builder.build(file) builder.build(file)
} }
fn writer_into_reader(writer: Writer<File>) -> anyhow::Result<Reader<Mmap>> { fn writer_into_reader(writer: Writer<File>, shrink_size: u64) -> anyhow::Result<Reader<FileFuse>> {
let file = writer.into_inner()?; let mut file = writer.into_inner()?;
let mmap = unsafe { Mmap::map(&file)? }; file.seek(SeekFrom::Start(0))?;
Reader::new(mmap).map_err(Into::into) let file = FileFuse::with_shrink_size(file, shrink_size);
Reader::new(file).map_err(Into::into)
} }
fn create_sorter( fn create_sorter(
merge: MergeFn, merge: MergeFn,
chunk_compression_type: CompressionType, chunk_compression_type: CompressionType,
chunk_compression_level: Option<u32>, chunk_compression_level: Option<u32>,
file_fusing_shrink_size: u64,
max_nb_chunks: Option<usize>, max_nb_chunks: Option<usize>,
max_memory: Option<usize>, max_memory: Option<usize>,
) -> Sorter<MergeFn> ) -> Sorter<MergeFn>
{ {
let mut builder = Sorter::builder(merge); let mut builder = Sorter::builder(merge);
builder.file_fusing_shrink_size(file_fusing_shrink_size);
builder.chunk_compression_type(chunk_compression_type); builder.chunk_compression_type(chunk_compression_type);
if let Some(level) = chunk_compression_level { if let Some(level) = chunk_compression_level {
builder.chunk_compression_level(level); builder.chunk_compression_level(level);
@ -194,11 +199,11 @@ fn compute_words_pair_proximities(
type MergeFn = fn(&[u8], &[Vec<u8>]) -> Result<Vec<u8>, ()>; type MergeFn = fn(&[u8], &[Vec<u8>]) -> Result<Vec<u8>, ()>;
struct Readers { struct Readers {
main: Reader<Mmap>, main: Reader<FileFuse>,
word_docids: Reader<Mmap>, word_docids: Reader<FileFuse>,
docid_word_positions: Reader<Mmap>, docid_word_positions: Reader<FileFuse>,
words_pairs_proximities_docids: Reader<Mmap>, words_pairs_proximities_docids: Reader<FileFuse>,
documents: Reader<Mmap>, documents: Reader<FileFuse>,
} }
struct Store { struct Store {
@ -210,6 +215,7 @@ struct Store {
// MTBL parameters // MTBL parameters
chunk_compression_type: CompressionType, chunk_compression_type: CompressionType,
chunk_compression_level: Option<u32>, chunk_compression_level: Option<u32>,
file_fusing_shrink_size: u64,
// MTBL sorters // MTBL sorters
main_sorter: Sorter<MergeFn>, main_sorter: Sorter<MergeFn>,
word_docids_sorter: Sorter<MergeFn>, word_docids_sorter: Sorter<MergeFn>,
@ -226,6 +232,7 @@ impl Store {
max_memory: Option<usize>, max_memory: Option<usize>,
chunk_compression_type: CompressionType, chunk_compression_type: CompressionType,
chunk_compression_level: Option<u32>, chunk_compression_level: Option<u32>,
file_fusing_shrink_size: u64,
) -> anyhow::Result<Store> ) -> anyhow::Result<Store>
{ {
// We divide the max memory by the number of sorter the Store have. // We divide the max memory by the number of sorter the Store have.
@ -235,6 +242,7 @@ impl Store {
main_merge, main_merge,
chunk_compression_type, chunk_compression_type,
chunk_compression_level, chunk_compression_level,
file_fusing_shrink_size,
max_nb_chunks, max_nb_chunks,
max_memory, max_memory,
); );
@ -242,6 +250,7 @@ impl Store {
word_docids_merge, word_docids_merge,
chunk_compression_type, chunk_compression_type,
chunk_compression_level, chunk_compression_level,
file_fusing_shrink_size,
max_nb_chunks, max_nb_chunks,
max_memory, max_memory,
); );
@ -249,14 +258,15 @@ impl Store {
words_pairs_proximities_docids_merge, words_pairs_proximities_docids_merge,
chunk_compression_type, chunk_compression_type,
chunk_compression_level, chunk_compression_level,
file_fusing_shrink_size,
max_nb_chunks, max_nb_chunks,
max_memory, max_memory,
); );
let documents_writer = tempfile().map(|f| { let documents_writer = tempfile().and_then(|f| {
create_writer(chunk_compression_type, chunk_compression_level, f) create_writer(chunk_compression_type, chunk_compression_level, f)
})?; })?;
let docid_word_positions_writer = tempfile().map(|f| { let docid_word_positions_writer = tempfile().and_then(|f| {
create_writer(chunk_compression_type, chunk_compression_level, f) create_writer(chunk_compression_type, chunk_compression_level, f)
})?; })?;
@ -268,6 +278,7 @@ impl Store {
documents_ids: RoaringBitmap::new(), documents_ids: RoaringBitmap::new(),
chunk_compression_type, chunk_compression_type,
chunk_compression_level, chunk_compression_level,
file_fusing_shrink_size,
main_sorter, main_sorter,
word_docids_sorter, word_docids_sorter,
@ -510,6 +521,7 @@ impl Store {
fn finish(mut self) -> anyhow::Result<Readers> { fn finish(mut self) -> anyhow::Result<Readers> {
let comp_type = self.chunk_compression_type; let comp_type = self.chunk_compression_type;
let comp_level = self.chunk_compression_level; let comp_level = self.chunk_compression_level;
let shrink_size = self.file_fusing_shrink_size;
Self::write_word_docids(&mut self.word_docids_sorter, self.word_docids)?; Self::write_word_docids(&mut self.word_docids_sorter, self.word_docids)?;
Self::write_documents_ids(&mut self.main_sorter, self.documents_ids)?; Self::write_documents_ids(&mut self.main_sorter, self.documents_ids)?;
@ -518,12 +530,11 @@ impl Store {
self.words_pairs_proximities_docids, self.words_pairs_proximities_docids,
)?; )?;
let mut word_docids_wtr = tempfile().map(|f| create_writer(comp_type, comp_level, f))?; let mut word_docids_wtr = tempfile().and_then(|f| create_writer(comp_type, comp_level, f))?;
let mut builder = fst::SetBuilder::memory(); let mut builder = fst::SetBuilder::memory();
let mut iter = self.word_docids_sorter.into_iter()?; let mut iter = self.word_docids_sorter.into_iter()?;
while let Some(result) = iter.next() { while let Some((word, val)) = iter.next()? {
let (word, val) = result?;
// This is a lexicographically ordered word position // This is a lexicographically ordered word position
// we use the key to construct the words fst. // we use the key to construct the words fst.
builder.insert(word)?; builder.insert(word)?;
@ -533,17 +544,17 @@ impl Store {
let fst = builder.into_set(); let fst = builder.into_set();
self.main_sorter.insert(WORDS_FST_KEY, fst.as_fst().as_bytes())?; self.main_sorter.insert(WORDS_FST_KEY, fst.as_fst().as_bytes())?;
let mut main_wtr = tempfile().map(|f| create_writer(comp_type, comp_level, f))?; let mut main_wtr = tempfile().and_then(|f| create_writer(comp_type, comp_level, f))?;
self.main_sorter.write_into(&mut main_wtr)?; self.main_sorter.write_into(&mut main_wtr)?;
let mut words_pairs_proximities_docids_wtr = tempfile().map(|f| create_writer(comp_type, comp_level, f))?; let mut words_pairs_proximities_docids_wtr = tempfile().and_then(|f| create_writer(comp_type, comp_level, f))?;
self.words_pairs_proximities_docids_sorter.write_into(&mut words_pairs_proximities_docids_wtr)?; self.words_pairs_proximities_docids_sorter.write_into(&mut words_pairs_proximities_docids_wtr)?;
let main = writer_into_reader(main_wtr)?; let main = writer_into_reader(main_wtr, shrink_size)?;
let word_docids = writer_into_reader(word_docids_wtr)?; let word_docids = writer_into_reader(word_docids_wtr, shrink_size)?;
let words_pairs_proximities_docids = writer_into_reader(words_pairs_proximities_docids_wtr)?; let words_pairs_proximities_docids = writer_into_reader(words_pairs_proximities_docids_wtr, shrink_size)?;
let docid_word_positions = writer_into_reader(self.docid_word_positions_writer)?; let docid_word_positions = writer_into_reader(self.docid_word_positions_writer, shrink_size)?;
let documents = writer_into_reader(self.documents_writer)?; let documents = writer_into_reader(self.documents_writer, shrink_size)?;
Ok(Readers { Ok(Readers {
main, main,
@ -614,7 +625,7 @@ fn documents_merge(key: &[u8], _values: &[Vec<u8>]) -> Result<Vec<u8>, ()> {
panic!("merging documents is an error ({:?})", key.as_bstr()) panic!("merging documents is an error ({:?})", key.as_bstr())
} }
fn merge_readers(sources: Vec<Reader<Mmap>>, merge: MergeFn) -> Merger<Mmap, MergeFn> { fn merge_readers(sources: Vec<Reader<FileFuse>>, merge: MergeFn) -> Merger<FileFuse, MergeFn> {
let mut builder = Merger::builder(merge); let mut builder = Merger::builder(merge);
builder.extend(sources); builder.extend(sources);
builder.build() builder.build()
@ -623,7 +634,7 @@ fn merge_readers(sources: Vec<Reader<Mmap>>, merge: MergeFn) -> Merger<Mmap, Mer
fn merge_into_lmdb_database( fn merge_into_lmdb_database(
wtxn: &mut heed::RwTxn, wtxn: &mut heed::RwTxn,
database: heed::PolyDatabase, database: heed::PolyDatabase,
sources: Vec<Reader<Mmap>>, sources: Vec<Reader<FileFuse>>,
merge: MergeFn, merge: MergeFn,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
debug!("Merging {} MTBL stores...", sources.len()); debug!("Merging {} MTBL stores...", sources.len());
@ -633,8 +644,7 @@ fn merge_into_lmdb_database(
let mut in_iter = merger.into_merge_iter()?; let mut in_iter = merger.into_merge_iter()?;
let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?; let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
while let Some(result) = in_iter.next() { while let Some((k, v)) = in_iter.next()? {
let (k, v) = result?;
out_iter.append(k, v).with_context(|| format!("writing {:?} into LMDB", k.as_bstr()))?; out_iter.append(k, v).with_context(|| format!("writing {:?} into LMDB", k.as_bstr()))?;
} }
@ -645,15 +655,13 @@ fn merge_into_lmdb_database(
fn write_into_lmdb_database( fn write_into_lmdb_database(
wtxn: &mut heed::RwTxn, wtxn: &mut heed::RwTxn,
database: heed::PolyDatabase, database: heed::PolyDatabase,
reader: Reader<Mmap>, mut reader: Reader<FileFuse>,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
debug!("Writing MTBL stores..."); debug!("Writing MTBL stores...");
let before = Instant::now(); let before = Instant::now();
let mut in_iter = reader.into_iter()?;
let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?; let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
while let Some(result) = in_iter.next() { while let Some((k, v)) = reader.next()? {
let (k, v) = result?;
out_iter.append(k, v).with_context(|| format!("writing {:?} into LMDB", k.as_bstr()))?; out_iter.append(k, v).with_context(|| format!("writing {:?} into LMDB", k.as_bstr()))?;
} }
@ -746,6 +754,7 @@ fn main() -> anyhow::Result<()> {
let max_memory_by_job = opt.indexer.max_memory / num_threads; let max_memory_by_job = opt.indexer.max_memory / num_threads;
let chunk_compression_type = opt.indexer.chunk_compression_type; let chunk_compression_type = opt.indexer.chunk_compression_type;
let chunk_compression_level = opt.indexer.chunk_compression_level; let chunk_compression_level = opt.indexer.chunk_compression_level;
let file_fusing_shrink_size = opt.indexer.file_fusing_shrink_size;
let log_every_n = opt.indexer.log_every_n; let log_every_n = opt.indexer.log_every_n;
let readers = csv_readers(opt.csv_file, num_threads)? let readers = csv_readers(opt.csv_file, num_threads)?
@ -758,6 +767,7 @@ fn main() -> anyhow::Result<()> {
Some(max_memory_by_job), Some(max_memory_by_job),
chunk_compression_type, chunk_compression_type,
chunk_compression_level, chunk_compression_level,
file_fusing_shrink_size,
)?; )?;
store.index_csv(rdr, i, num_threads, log_every_n) store.index_csv(rdr, i, num_threads, log_every_n)
}) })
@ -779,12 +789,12 @@ fn main() -> anyhow::Result<()> {
// This is the function that merge the readers // This is the function that merge the readers
// by using the given merge function. // by using the given merge function.
let merge_readers = move |readers, merge| { let merge_readers = move |readers, merge| {
let mut writer = tempfile().map(|f| { let mut writer = tempfile().and_then(|f| {
create_writer(chunk_compression_type, chunk_compression_level, f) create_writer(chunk_compression_type, chunk_compression_level, f)
})?; })?;
let merger = merge_readers(readers, merge); let merger = merge_readers(readers, merge);
merger.write_into(&mut writer)?; merger.write_into(&mut writer)?;
writer_into_reader(writer) writer_into_reader(writer, file_fusing_shrink_size)
}; };
// The enum and the channel which is used to transfert // The enum and the channel which is used to transfert