From f5a4a1c8b26a37694716cd6a44d3c3610d4239b4 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 23 Jan 2025 10:55:03 +0100 Subject: [PATCH] Give more RAM to bbqueue. - bbqueue buffers used to have (5% * 2%) / num_threads - they now have 5% / num_threads --- crates/milli/src/update/new/indexer/mod.rs | 24 ++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index 1cf83f2d2..b65750030 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -68,17 +68,25 @@ where ..grenad_parameters }; - // We compute and remove the allocated BBQueues buffers capacity from the indexing memory. - let minimum_capacity = 50 * 1024 * 1024 * pool.current_num_threads(); // 50 MiB + // 5% percent of the allocated memory for the extractors, or min 100MiB + // 5% percent of the allocated memory for the bbqueues, or min 50MiB + // + // Minimum capacity for bbqueues + let minimum_total_bbbuffer_capacity = 50 * 1024 * 1024 * pool.current_num_threads(); // 50 MiB + let minimum_total_extractors_capacity = minimum_total_bbbuffer_capacity * 2; + let (grenad_parameters, total_bbbuffer_capacity) = grenad_parameters.max_memory.map_or( - (grenad_parameters, 2 * minimum_capacity), // 100 MiB by thread by default + ( + GrenadParameters { + max_memory: Some(minimum_total_extractors_capacity), + ..grenad_parameters + }, + minimum_total_bbbuffer_capacity, + ), // 100 MiB by thread by default |max_memory| { - // 2% of the indexing memory - let total_bbbuffer_capacity = (max_memory / 100 / 2).max(minimum_capacity); + let total_bbbuffer_capacity = max_memory.max(minimum_total_bbbuffer_capacity); let new_grenad_parameters = GrenadParameters { - max_memory: Some( - max_memory.saturating_sub(total_bbbuffer_capacity).max(100 * 1024 * 1024), - ), + max_memory: Some(max_memory.max(minimum_total_extractors_capacity)), ..grenad_parameters }; (new_grenad_parameters, total_bbbuffer_capacity)