snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 20:15:07 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Always sort internal Sorter entries in parallel

Browse Source
This commit is contained in:
Clément Renault 2023-11-02 14:47:43 +01:00
parent b10c060bf7
commit 4d864f0702
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
2 changed files with 17 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
milli/src/update/index_documents/helpers/grenad_helpers.rs
View File

@ -47,6 +47,7 @@ pub fn create_sorter(
builder.allow_realloc(false); builder.allow_realloc(false);
} }
builder.sort_algorithm(sort_algorithm); builder.sort_algorithm(sort_algorithm);
builder.sort_in_parallel(true);
builder.build() builder.build()
} }

51
milli/src/update/index_documents/transform.rs
View File

@ -114,43 +114,24 @@ impl<'a, 'i> Transform<'a, 'i> {
}; };
// We initialize the sorter with the user indexing settings. // We initialize the sorter with the user indexing settings.
let original_sorter = { let original_sorter = create_sorter(
let mut builder = grenad::Sorter::builder(merge_function); grenad::SortAlgorithm::Stable,
builder.chunk_compression_type(indexer_settings.chunk_compression_type); merge_function,
if let Some(level) = indexer_settings.chunk_compression_level { indexer_settings.chunk_compression_type,
builder.chunk_compression_level(level); indexer_settings.chunk_compression_level,
} indexer_settings.max_nb_chunks,
if let Some(nb_chunks) = indexer_settings.max_nb_chunks { indexer_settings.max_memory.map(|mem| mem / 2),
builder.max_nb_chunks(nb_chunks); );
}
if let Some(memory) = indexer_settings.max_memory.map(|mem| mem / 2) {
builder.dump_threshold(memory);
builder.allow_realloc(false);
}
builder.sort_algorithm(grenad::SortAlgorithm::Stable);
builder.sort_in_parallel(true);
builder.build()
};
// We initialize the sorter with the user indexing settings. // We initialize the sorter with the user indexing settings.
let flattened_sorter = { let flattened_sorter = create_sorter(
let mut builder = grenad::Sorter::builder(merge_function); grenad::SortAlgorithm::Stable,
builder.chunk_compression_type(indexer_settings.chunk_compression_type); merge_function,
if let Some(level) = indexer_settings.chunk_compression_level { indexer_settings.chunk_compression_type,
builder.chunk_compression_level(level); indexer_settings.chunk_compression_level,
} indexer_settings.max_nb_chunks,
if let Some(nb_chunks) = indexer_settings.max_nb_chunks { indexer_settings.max_memory.map(|mem| mem / 2),
builder.max_nb_chunks(nb_chunks); );
}
if let Some(memory) = indexer_settings.max_memory.map(|mem| mem / 2) {
builder.dump_threshold(memory);
builder.allow_realloc(false);
}
builder.sort_algorithm(grenad::SortAlgorithm::Stable);
builder.sort_in_parallel(true);
builder.build()
};
let documents_ids = index.documents_ids(wtxn)?; let documents_ids = index.documents_ids(wtxn)?;
Ok(Transform { Ok(Transform {