snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 02:27:40 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Merge #283

Browse Source 
283: Use the AlwaysFreePages flag when opening an index r=irevoire a=Kerollmops

We introduced a new flag in our fork of LMDB, this `AlwaysFreePages` flag forces LMDB to always free the single pages it uses before writing to the disk instead of keeping them in a linked list.

Declaring this flag reduces the memory print (leak) we have on memory after indexing a lot of documents.

Fixes #279.

Co-authored-by: Kerollmops <clement@meilisearch.com>
This commit is contained in:
bors[bot] 2021-07-05 16:59:16 +00:00 committed by GitHub
commit cc54c41e30
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.lock generated
View File

@ -873,7 +873,7 @@ dependencies = [
[[package]]
name = "heed"
version = "0.12.0"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
dependencies = [
"byteorder",
"heed-traits",
@ -891,12 +891,12 @@ dependencies = [
[[package]]
name = "heed-traits"
version = "0.7.0"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
[[package]]
name = "heed-types"
version = "0.7.2"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
dependencies = [
"bincode",
"heed-traits",

2
benchmarks/Cargo.toml
View File

@ -11,7 +11,7 @@ milli = { path = "../milli" }
jemallocator = "0.3.2"
[dev-dependencies]
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
criterion = { version = "0.3.4", features = ["html_reports"] }
[build-dependencies]

2
helpers/Cargo.toml
View File

@ -7,7 +7,7 @@ edition = "2018"
[dependencies]
anyhow = "1.0.38"
byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
milli = { path = "../milli" }
stderrlog = "0.5.1"
structopt = { version = "0.3.21", default-features = false }

2
http-ui/Cargo.toml
View File

@ -10,7 +10,7 @@ anyhow = "1.0.38"
byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
crossbeam-channel = "0.5.0"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" }
memmap = "0.7.0"
milli = { path = "../milli" }

2
infos/Cargo.toml
View File

@ -8,7 +8,7 @@ edition = "2018"
anyhow = "1.0.38"
byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
csv = "1.1.5"
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
milli = { path = "../milli" }
roaring = "0.6.6"
serde_json = "1.0.62"

2
milli/Cargo.toml
View File

@ -14,7 +14,7 @@ flate2 = "1.0.20"
fst = "0.4.5"
fxhash = "0.2.1"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0", default-features = false, features = ["lmdb", "sync-read-txn"] }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] }
human_format = "1.0.3"
levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
linked-hash-map = "0.5.4"

2
milli/src/index.rs
View File

@ -3,6 +3,7 @@ use std::collections::{HashMap, HashSet};
use std::path::Path;
use chrono::{DateTime, Utc};
use heed::flags::Flags;
use heed::types::*;
use heed::{Database, PolyDatabase, RoTxn, RwTxn};
use roaring::RoaringBitmap;
@ -106,6 +107,7 @@ impl Index {
use db_name::*;
options.max_dbs(14);
unsafe { options.flag(Flags::MdbAlwaysFreePages) };
let env = options.open(path)?;
let main = env.create_poly_database(Some(MAIN))?;

2
search/Cargo.toml
View File

@ -7,7 +7,7 @@ edition = "2018"
[dependencies]
anyhow = "1.0.38"
byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
log = "0.4.14"
milli = { path = "../milli" }
serde_json = "1.0.62"