From 4d864f0702578e6540207c1472992fab06d63b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 2 Nov 2023 14:47:43 +0100 Subject: [PATCH] Always sort internal Sorter entries in parallel --- .../index_documents/helpers/grenad_helpers.rs | 1 + milli/src/update/index_documents/transform.rs | 51 ++++++------------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/milli/src/update/index_documents/helpers/grenad_helpers.rs b/milli/src/update/index_documents/helpers/grenad_helpers.rs index cc0ccb609..03a3d6f5f 100644 --- a/milli/src/update/index_documents/helpers/grenad_helpers.rs +++ b/milli/src/update/index_documents/helpers/grenad_helpers.rs @@ -47,6 +47,7 @@ pub fn create_sorter( builder.allow_realloc(false); } builder.sort_algorithm(sort_algorithm); + builder.sort_in_parallel(true); builder.build() } diff --git a/milli/src/update/index_documents/transform.rs b/milli/src/update/index_documents/transform.rs index 8d1750c49..23b5c78c1 100644 --- a/milli/src/update/index_documents/transform.rs +++ b/milli/src/update/index_documents/transform.rs @@ -114,43 +114,24 @@ impl<'a, 'i> Transform<'a, 'i> { }; // We initialize the sorter with the user indexing settings. - let original_sorter = { - let mut builder = grenad::Sorter::builder(merge_function); - builder.chunk_compression_type(indexer_settings.chunk_compression_type); - if let Some(level) = indexer_settings.chunk_compression_level { - builder.chunk_compression_level(level); - } - if let Some(nb_chunks) = indexer_settings.max_nb_chunks { - builder.max_nb_chunks(nb_chunks); - } - if let Some(memory) = indexer_settings.max_memory.map(|mem| mem / 2) { - builder.dump_threshold(memory); - builder.allow_realloc(false); - } - builder.sort_algorithm(grenad::SortAlgorithm::Stable); - builder.sort_in_parallel(true); - builder.build() - }; + let original_sorter = create_sorter( + grenad::SortAlgorithm::Stable, + merge_function, + indexer_settings.chunk_compression_type, + indexer_settings.chunk_compression_level, + indexer_settings.max_nb_chunks, + indexer_settings.max_memory.map(|mem| mem / 2), + ); // We initialize the sorter with the user indexing settings. - let flattened_sorter = { - let mut builder = grenad::Sorter::builder(merge_function); - builder.chunk_compression_type(indexer_settings.chunk_compression_type); - if let Some(level) = indexer_settings.chunk_compression_level { - builder.chunk_compression_level(level); - } - if let Some(nb_chunks) = indexer_settings.max_nb_chunks { - builder.max_nb_chunks(nb_chunks); - } - if let Some(memory) = indexer_settings.max_memory.map(|mem| mem / 2) { - builder.dump_threshold(memory); - builder.allow_realloc(false); - } - builder.sort_algorithm(grenad::SortAlgorithm::Stable); - builder.sort_in_parallel(true); - builder.build() - }; - + let flattened_sorter = create_sorter( + grenad::SortAlgorithm::Stable, + merge_function, + indexer_settings.chunk_compression_type, + indexer_settings.chunk_compression_level, + indexer_settings.max_nb_chunks, + indexer_settings.max_memory.map(|mem| mem / 2), + ); let documents_ids = index.documents_ids(wtxn)?; Ok(Transform {