From 8380ddbdcd888ec0c16401af991c8ccbed9a3d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 20 Nov 2024 15:10:09 +0100 Subject: [PATCH] Fix progress of into_changes --- crates/benchmarks/benches/indexing.rs | 30 +++++++++++++++++++ crates/benchmarks/benches/utils.rs | 10 ++++++- crates/fuzzers/src/bin/fuzz-indexing.rs | 1 + crates/index-scheduler/src/batch.rs | 1 + crates/milli/src/index.rs | 3 ++ .../milli/src/search/new/tests/integration.rs | 10 ++++++- .../milli/src/update/index_documents/mod.rs | 12 ++++++++ .../milli/tests/search/facet_distribution.rs | 10 ++++++- crates/milli/tests/search/mod.rs | 10 ++++++- crates/milli/tests/search/query_criteria.rs | 10 ++++++- crates/milli/tests/search/typo_tolerance.rs | 10 ++++++- 11 files changed, 101 insertions(+), 6 deletions(-) diff --git a/crates/benchmarks/benches/indexing.rs b/crates/benchmarks/benches/indexing.rs index 3210a6e7d..2f33c3454 100644 --- a/crates/benchmarks/benches/indexing.rs +++ b/crates/benchmarks/benches/indexing.rs @@ -150,6 +150,7 @@ fn indexing_songs_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -215,6 +216,7 @@ fn reindexing_songs_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -258,6 +260,7 @@ fn reindexing_songs_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -325,6 +328,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -400,6 +404,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -443,6 +448,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -482,6 +488,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -548,6 +555,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -613,6 +621,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -678,6 +687,7 @@ fn indexing_wiki(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -742,6 +752,7 @@ fn reindexing_wiki(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -785,6 +796,7 @@ fn reindexing_wiki(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -851,6 +863,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -926,6 +939,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -970,6 +984,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1010,6 +1025,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1075,6 +1091,7 @@ fn indexing_movies_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1139,6 +1156,7 @@ fn reindexing_movies_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1182,6 +1200,7 @@ fn reindexing_movies_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1248,6 +1267,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1358,6 +1378,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1401,6 +1422,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1440,6 +1462,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1528,6 +1551,7 @@ fn indexing_nested_movies_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1617,6 +1641,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1698,6 +1723,7 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1763,6 +1789,7 @@ fn indexing_geo(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1827,6 +1854,7 @@ fn reindexing_geo(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1870,6 +1898,7 @@ fn reindexing_geo(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -1936,6 +1965,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); diff --git a/crates/benchmarks/benches/utils.rs b/crates/benchmarks/benches/utils.rs index 09a932c65..6a08b96b5 100644 --- a/crates/benchmarks/benches/utils.rs +++ b/crates/benchmarks/benches/utils.rs @@ -103,7 +103,15 @@ pub fn base_setup(conf: &Conf) -> Index { let indexer_alloc = Bump::new(); let (document_changes, _operation_stats, primary_key) = indexer - .into_changes(&indexer_alloc, &index, &rtxn, None, &mut new_fields_ids_map, &|| false) + .into_changes( + &indexer_alloc, + &index, + &rtxn, + None, + &mut new_fields_ids_map, + &|| false, + &|_progress| (), + ) .unwrap(); indexer::index( diff --git a/crates/fuzzers/src/bin/fuzz-indexing.rs b/crates/fuzzers/src/bin/fuzz-indexing.rs index 6b57d296f..f335938b9 100644 --- a/crates/fuzzers/src/bin/fuzz-indexing.rs +++ b/crates/fuzzers/src/bin/fuzz-indexing.rs @@ -128,6 +128,7 @@ fn main() { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); diff --git a/crates/index-scheduler/src/batch.rs b/crates/index-scheduler/src/batch.rs index 13ba70cc6..75b092b28 100644 --- a/crates/index-scheduler/src/batch.rs +++ b/crates/index-scheduler/src/batch.rs @@ -1344,6 +1344,7 @@ impl IndexScheduler { primary_key.as_deref(), &mut new_fields_ids_map, &|| must_stop_processing.get(), + &send_progress, )?; let mut addition = 0; diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index df30f678a..89f965b7c 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1764,6 +1764,7 @@ pub(crate) mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), )?; if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { @@ -1853,6 +1854,7 @@ pub(crate) mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), )?; if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { @@ -1932,6 +1934,7 @@ pub(crate) mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); diff --git a/crates/milli/src/search/new/tests/integration.rs b/crates/milli/src/search/new/tests/integration.rs index 9872acbe7..79668b34b 100644 --- a/crates/milli/src/search/new/tests/integration.rs +++ b/crates/milli/src/search/new/tests/integration.rs @@ -65,7 +65,15 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let indexer_alloc = Bump::new(); let (document_changes, operation_stats, primary_key) = indexer - .into_changes(&indexer_alloc, &index, &rtxn, None, &mut new_fields_ids_map, &|| false) + .into_changes( + &indexer_alloc, + &index, + &rtxn, + None, + &mut new_fields_ids_map, + &|| false, + &|_progress| (), + ) .unwrap(); if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index d845d8c35..baecbdcf0 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -1947,6 +1947,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2130,6 +2131,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2190,6 +2192,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2241,6 +2244,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2291,6 +2295,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2343,6 +2348,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2400,6 +2406,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2450,6 +2457,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2500,6 +2508,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2692,6 +2701,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2749,6 +2759,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); @@ -2803,6 +2814,7 @@ mod tests { None, &mut new_fields_ids_map, &|| false, + &|_progress| (), ) .unwrap(); diff --git a/crates/milli/tests/search/facet_distribution.rs b/crates/milli/tests/search/facet_distribution.rs index 0ec1c206f..61d0697ff 100644 --- a/crates/milli/tests/search/facet_distribution.rs +++ b/crates/milli/tests/search/facet_distribution.rs @@ -50,7 +50,15 @@ fn test_facet_distribution_with_no_facet_values() { let indexer_alloc = Bump::new(); let (document_changes, _operation_stats, primary_key) = indexer - .into_changes(&indexer_alloc, &index, &rtxn, None, &mut new_fields_ids_map, &|| false) + .into_changes( + &indexer_alloc, + &index, + &rtxn, + None, + &mut new_fields_ids_map, + &|| false, + &|_progress| (), + ) .unwrap(); indexer::index( diff --git a/crates/milli/tests/search/mod.rs b/crates/milli/tests/search/mod.rs index cee869a8d..1287b59d5 100644 --- a/crates/milli/tests/search/mod.rs +++ b/crates/milli/tests/search/mod.rs @@ -83,7 +83,15 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let indexer_alloc = Bump::new(); let (document_changes, operation_stats, primary_key) = indexer - .into_changes(&indexer_alloc, &index, &rtxn, None, &mut new_fields_ids_map, &|| false) + .into_changes( + &indexer_alloc, + &index, + &rtxn, + None, + &mut new_fields_ids_map, + &|| false, + &|_progress| (), + ) .unwrap(); if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { diff --git a/crates/milli/tests/search/query_criteria.rs b/crates/milli/tests/search/query_criteria.rs index 6cbf4f27d..3e56eeff0 100644 --- a/crates/milli/tests/search/query_criteria.rs +++ b/crates/milli/tests/search/query_criteria.rs @@ -319,7 +319,15 @@ fn criteria_ascdesc() { let payload = unsafe { memmap2::Mmap::map(&file).unwrap() }; indexer.add_documents(&payload).unwrap(); let (document_changes, _operation_stats, primary_key) = indexer - .into_changes(&indexer_alloc, &index, &rtxn, None, &mut new_fields_ids_map, &|| false) + .into_changes( + &indexer_alloc, + &index, + &rtxn, + None, + &mut new_fields_ids_map, + &|| false, + &|_progress| (), + ) .unwrap(); indexer::index( diff --git a/crates/milli/tests/search/typo_tolerance.rs b/crates/milli/tests/search/typo_tolerance.rs index 5da98a680..7ac9a1e4b 100644 --- a/crates/milli/tests/search/typo_tolerance.rs +++ b/crates/milli/tests/search/typo_tolerance.rs @@ -128,7 +128,15 @@ fn test_typo_disabled_on_word() { let indexer_alloc = Bump::new(); let (document_changes, _operation_stats, primary_key) = indexer - .into_changes(&indexer_alloc, &index, &rtxn, None, &mut new_fields_ids_map, &|| false) + .into_changes( + &indexer_alloc, + &index, + &rtxn, + None, + &mut new_fields_ids_map, + &|| false, + &|_progress| (), + ) .unwrap(); indexer::index(