snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 20:15:07 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge #602

Browse Source 
602: Use mimalloc as the default allocator r=Kerollmops a=loiclec

## What does this PR do?
Use mimalloc as the global allocator for milli's benchmarks on macOS.

## Why?
On Linux, we use jemalloc, which is a very fast allocator. But on macOS, we currently use the system allocator, which is very slow. In practice, this difference in allocator speed means that it is difficult to gain insight into milli's performance by running benchmarks locally on the Mac.

By using mimalloc, which is another excellent allocator, we reduce the speed difference between the two platforms.

Co-authored-by: Loïc Lecrenier <loic@meilisearch.com>
This commit is contained in:
bors[bot] 2022-08-17 10:26:13 +00:00 committed by GitHub
commit cd2635ccfc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 30 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
benchmarks/Cargo.toml
View File

@ -5,17 +5,15 @@ edition = "2018"
publish = false
[dependencies]
milli = { path = "../milli" }
anyhow = "1.0.56"
serde_json = { version = "1.0.79", features = ["preserve_order"] }
csv = "1.1.6"
[target.'cfg(target_os = "linux")'.dependencies]
jemallocator = "0.3.2"
milli = { path = "../milli" }
mimalloc = { version = "0.1.29", default-features = false }
serde_json = { version = "1.0.79", features = ["preserve_order"] }
[dev-dependencies]
heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" }
criterion = { version = "0.3.5", features = ["html_reports"] }
heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" }
rand = "0.8.5"
rand_chacha = "0.3.1"
roaring = "0.9.0"
@ -23,8 +21,8 @@ roaring = "0.9.0"
[build-dependencies]
anyhow = "1.0.56"
bytes = "1.1.0"
flate2 = "1.0.22"
convert_case = "0.5.0"
flate2 = "1.0.22"
reqwest = { version = "0.11.9", features = ["blocking", "rustls-tls"], default-features = false }
[[bench]]

3
benchmarks/benches/formatting.rs
View File

@ -2,9 +2,8 @@ use criterion::{criterion_group, criterion_main};
use milli::tokenizer::TokenizerBuilder;
use milli::{FormatOptions, MatcherBuilder, MatchingWord, MatchingWords};
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
struct Conf<'a> {
name: &'a str,

3
benchmarks/benches/indexing.rs
View File

@ -14,9 +14,8 @@ use rand::seq::SliceRandom;
use rand_chacha::rand_core::SeedableRng;
use roaring::RoaringBitmap;
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
const BENCHMARK_ITERATION: usize = 10;

3
benchmarks/benches/search_geo.rs
View File

@ -5,9 +5,8 @@ use criterion::{criterion_group, criterion_main};
use milli::update::Settings;
use utils::Conf;
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
fn base_conf(builder: &mut Settings) {
let displayed_fields =

3
benchmarks/benches/search_songs.rs
View File

@ -5,9 +5,8 @@ use criterion::{criterion_group, criterion_main};
use milli::update::Settings;
use utils::Conf;
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
fn base_conf(builder: &mut Settings) {
let displayed_fields =

3
benchmarks/benches/search_wiki.rs
View File

@ -5,9 +5,8 @@ use criterion::{criterion_group, criterion_main};
use milli::update::Settings;
use utils::Conf;
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
fn base_conf(builder: &mut Settings) {
let displayed_fields = ["title", "body", "url"].iter().map(|s| s.to_string()).collect();

20
cli/Cargo.toml
View File

@ -6,18 +6,16 @@ description = "A CLI to interact with a milli index"
publish = false
[dependencies]
bimap = "0.6.2"
byte-unit = { version = "4.0.14", features = ["serde"] }
color-eyre = "0.6.1"
csv = "1.1.6"
eyre = "0.6.7"
heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] }
indicatif = "0.16.2"
milli = { path = "../milli" }
mimalloc = { version = "0.1.29", default-features = false }
serde = "1.0.136"
serde_json = "1.0.79"
structopt = "0.3.26"
milli = { path = "../milli" }
eyre = "0.6.7"
color-eyre = "0.6.1"
heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] }
byte-unit = { version = "4.0.14", features = ["serde"] }
bimap = "0.6.2"
csv = "1.1.6"
stderrlog = "0.5.1"
[target.'cfg(target_os = "linux")'.dependencies]
jemallocator = "0.3.2"
structopt = "0.3.26"

3
cli/src/main.rs
View File

@ -16,9 +16,8 @@ use milli::update::{self, IndexDocumentsConfig, IndexDocumentsMethod, IndexerCon
use milli::{Index, Object};
use structopt::StructOpt;
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
#[derive(Debug, StructOpt)]
#[structopt(name = "Milli CLI", about = "A simple CLI to manipulate a milli index.")]

4
helpers/Cargo.toml
View File

@ -11,8 +11,6 @@ anyhow = "1.0.56"
byte-unit = { version = "4.0.14", default-features = false, features = ["std"] }
heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" }
milli = { path = "../milli" }
mimalloc = { version = "0.1.29", default-features = false }
stderrlog = "0.5.1"
structopt = { version = "0.3.26", default-features = false }
[target.'cfg(target_os = "linux")'.dependencies]
jemallocator = "0.3.2"

3
helpers/src/main.rs
View File

@ -5,9 +5,8 @@ use heed::{CompactionOption, Env, EnvOpenOptions};
use structopt::StructOpt;
use Command::*;
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
#[derive(Debug, StructOpt)]
/// Some helpers commands for milli.

7
http-ui/Cargo.toml
View File

@ -13,6 +13,7 @@ crossbeam-channel = "0.5.2"
heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" }
memmap2 = "0.5.3"
milli = { path = "../milli" }
mimalloc = { version = "0.1.29", default-features = false }
once_cell = "1.10.0"
rayon = "1.5.1"
structopt = { version = "0.3.26", default-features = false, features = ["wrap_help"] }
@ -32,18 +33,16 @@ tokio-stream = { version = "0.1.8", default-features = false, features = ["sync"
warp = "0.3.2"
# logging
fst = "0.4.7"
log = "0.4.14"
stderrlog = "0.5.1"
fst = "0.4.7"
# Temporary fix for bitvec, remove once fixed. (https://github.com/bitvecto-rs/bitvec/issues/105)
funty = "2.0.0"
bimap = "0.6.2"
csv = "1.1.6"
funty = "2.0.0"
[dev-dependencies]
maplit = "1.0.2"
serde_test = "1.0.136"
[target.'cfg(target_os = "linux")'.dependencies]
jemallocator = "0.3.2"

3
http-ui/src/main.rs
View File

@ -42,9 +42,8 @@ use warp::Filter;
use self::update_store::UpdateStore;
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
static GLOBAL_CONFIG: OnceCell<IndexerConfig> = OnceCell::new();

4
infos/Cargo.toml
View File

@ -11,10 +11,8 @@ byte-unit = { version = "4.0.14", default-features = false, features = ["std"] }
csv = "1.1.6"
heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.1" }
milli = { path = "../milli" }
mimalloc = { version = "0.1.29", default-features = false }
roaring = "0.9.0"
serde_json = "1.0.79"
stderrlog = "0.5.1"
structopt = { version = "0.3.26", default-features = false }
[target.'cfg(target_os = "linux")'.dependencies]
jemallocator = "0.3.2"

3
infos/src/main.rs
View File

@ -11,9 +11,8 @@ use milli::{FieldId, Index};
use structopt::StructOpt;
use Command::*;
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
const ALL_DATABASE_NAMES: &[&str] = &[
MAIN,

4
milli/fuzz/Cargo.toml
View File

@ -16,9 +16,7 @@ serde_json = { version = "1.0.62", features = ["preserve_order"] }
anyhow = "1.0"
tempfile = "3.3"
arbitrary-json = "0.1.0"
[target.'cfg(target_os = "linux")'.dependencies]
jemallocator = "0.3.2"
mimalloc = { version = "0.1.29", default-features = false }
[dependencies.milli]
path = ".."

3
milli/fuzz/fuzz_targets/indexing.rs
View File

@ -12,9 +12,8 @@ use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Setting
use milli::Index;
use serde_json::{Map, Value};
#[cfg(target_os = "linux")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
/// reads json from input and write an obkv batch to writer.
pub fn read_json(input: impl Read, writer: impl Write + Seek) -> Result<usize> {