From 6eb47ab792391667bfb9927db979469116a5f48a Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Wed, 3 Nov 2021 13:12:01 +0100 Subject: [PATCH] remove update_id in UpdateBuilder --- benchmarks/benches/indexing.rs | 48 ++++---- benchmarks/benches/utils.rs | 8 +- cli/src/main.rs | 8 +- http-ui/src/main.rs | 6 +- milli/src/index.rs | 12 +- milli/src/search/distinct/mod.rs | 8 +- milli/src/search/facet/filter.rs | 8 +- milli/src/update/clear_documents.rs | 13 +- milli/src/update/delete_documents.rs | 36 +++--- milli/src/update/facets.rs | 8 +- milli/src/update/index_documents/mod.rs | 114 +++++++++--------- milli/src/update/settings.rs | 152 +++++++++++------------- milli/src/update/update_builder.rs | 14 +-- milli/src/update/words_prefixes_fst.rs | 10 +- milli/tests/search/distinct.rs | 4 +- milli/tests/search/mod.rs | 8 +- milli/tests/search/query_criteria.rs | 16 +-- 17 files changed, 217 insertions(+), 256 deletions(-) diff --git a/benchmarks/benches/indexing.rs b/benchmarks/benches/indexing.rs index e4657d5b6..93a57091a 100644 --- a/benchmarks/benches/indexing.rs +++ b/benchmarks/benches/indexing.rs @@ -39,7 +39,7 @@ fn indexing_songs_default(c: &mut Criterion) { move || { let index = setup_index(); - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let mut builder = update_builder.settings(&mut wtxn, &index); @@ -61,17 +61,17 @@ fn indexing_songs_default(c: &mut Criterion) { .map(|s| s.to_string()) .collect(); builder.set_filterable_fields(faceted_fields); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); index }, move |index| { - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let builder = update_builder.index_documents(&mut wtxn, &index); let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv"); - builder.execute(documents, |_, _| ()).unwrap(); + builder.execute(documents, |_| ()).unwrap(); wtxn.commit().unwrap(); index.prepare_for_closing().wait(); @@ -88,7 +88,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) { move || { let index = setup_index(); - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let mut builder = update_builder.settings(&mut wtxn, &index); @@ -107,17 +107,17 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) { let faceted_fields = ["genre", "country", "artist"].iter().map(|s| s.to_string()).collect(); builder.set_filterable_fields(faceted_fields); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); index }, move |index| { - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let builder = update_builder.index_documents(&mut wtxn, &index); let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv"); - builder.execute(documents, |_, _| ()).unwrap(); + builder.execute(documents, |_| ()).unwrap(); wtxn.commit().unwrap(); index.prepare_for_closing().wait(); @@ -134,7 +134,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) { move || { let index = setup_index(); - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let mut builder = update_builder.settings(&mut wtxn, &index); @@ -149,17 +149,17 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) { let searchable_fields = ["title", "album", "artist"].iter().map(|s| s.to_string()).collect(); builder.set_searchable_fields(searchable_fields); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); index }, move |index| { - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let builder = update_builder.index_documents(&mut wtxn, &index); let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv"); - builder.execute(documents, |_, _| ()).unwrap(); + builder.execute(documents, |_| ()).unwrap(); wtxn.commit().unwrap(); index.prepare_for_closing().wait(); @@ -176,7 +176,7 @@ fn indexing_wiki(c: &mut Criterion) { move || { let index = setup_index(); - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let mut builder = update_builder.settings(&mut wtxn, &index); @@ -190,18 +190,18 @@ fn indexing_wiki(c: &mut Criterion) { // there is NO faceted fields at all - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); index }, move |index| { - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let mut builder = update_builder.index_documents(&mut wtxn, &index); builder.enable_autogenerate_docids(); let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv"); - builder.execute(documents, |_, _| ()).unwrap(); + builder.execute(documents, |_| ()).unwrap(); wtxn.commit().unwrap(); index.prepare_for_closing().wait(); @@ -218,7 +218,7 @@ fn indexing_movies_default(c: &mut Criterion) { move || { let index = setup_index(); - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let mut builder = update_builder.settings(&mut wtxn, &index); @@ -237,17 +237,17 @@ fn indexing_movies_default(c: &mut Criterion) { ["released_date", "genres"].iter().map(|s| s.to_string()).collect(); builder.set_filterable_fields(faceted_fields); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); index }, move |index| { - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let builder = update_builder.index_documents(&mut wtxn, &index); let documents = utils::documents_from(datasets_paths::MOVIES, "json"); - builder.execute(documents, |_, _| ()).unwrap(); + builder.execute(documents, |_| ()).unwrap(); wtxn.commit().unwrap(); index.prepare_for_closing().wait(); @@ -264,7 +264,7 @@ fn indexing_geo(c: &mut Criterion) { move || { let index = setup_index(); - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let mut builder = update_builder.settings(&mut wtxn, &index); @@ -288,17 +288,17 @@ fn indexing_geo(c: &mut Criterion) { ["_geo", "population", "elevation"].iter().map(|s| s.to_string()).collect(); builder.set_sortable_fields(sortable_fields); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); index }, move |index| { - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let builder = update_builder.index_documents(&mut wtxn, &index); let documents = utils::documents_from(datasets_paths::SMOL_ALL_COUNTRIES, "jsonl"); - builder.execute(documents, |_, _| ()).unwrap(); + builder.execute(documents, |_| ()).unwrap(); wtxn.commit().unwrap(); diff --git a/benchmarks/benches/utils.rs b/benchmarks/benches/utils.rs index 655275967..1b1d9be8c 100644 --- a/benchmarks/benches/utils.rs +++ b/benchmarks/benches/utils.rs @@ -65,7 +65,7 @@ pub fn base_setup(conf: &Conf) -> Index { options.max_readers(10); let index = Index::new(options, conf.database_name).unwrap(); - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let mut builder = update_builder.settings(&mut wtxn, &index); @@ -84,10 +84,10 @@ pub fn base_setup(conf: &Conf) -> Index { (conf.configure)(&mut builder); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); - let update_builder = UpdateBuilder::new(0); + let update_builder = UpdateBuilder::new(); let mut wtxn = index.write_txn().unwrap(); let mut builder = update_builder.index_documents(&mut wtxn, &index); if let None = conf.primary_key { @@ -96,7 +96,7 @@ pub fn base_setup(conf: &Conf) -> Index { let documents = documents_from(conf.dataset, conf.dataset_format); builder.index_documents_method(IndexDocumentsMethod::ReplaceDocuments); - builder.execute(documents, |_, _| ()).unwrap(); + builder.execute(documents, |_| ()).unwrap(); wtxn.commit().unwrap(); index diff --git a/cli/src/main.rs b/cli/src/main.rs index 5e11dc3fb..44c197de6 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -122,7 +122,7 @@ impl DocumentAddition { println!("Adding {} documents to the index.", reader.len()); let mut txn = index.env.write_txn()?; - let mut addition = milli::update::IndexDocuments::new(&mut txn, &index, 0); + let mut addition = milli::update::IndexDocuments::new(&mut txn, &index); if self.update_documents { addition.index_documents_method(milli::update::IndexDocumentsMethod::UpdateDocuments); @@ -146,7 +146,7 @@ impl DocumentAddition { progesses.join().unwrap(); }); - let result = addition.execute(reader, |step, _| indexing_callback(step, &bars))?; + let result = addition.execute(reader, |step| indexing_callback(step, &bars))?; txn.commit()?; @@ -292,7 +292,7 @@ impl SettingsUpdate { fn perform(&self, index: milli::Index) -> Result<()> { let mut txn = index.env.write_txn()?; - let mut update = milli::update::Settings::new(&mut txn, &index, 0); + let mut update = milli::update::Settings::new(&mut txn, &index); update.log_every_n(100); if let Some(ref filterable_attributes) = self.filterable_attributes { @@ -315,7 +315,7 @@ impl SettingsUpdate { progesses.join().unwrap(); }); - update.execute(|step, _| indexing_callback(step, &bars))?; + update.execute(|step| indexing_callback(step, &bars))?; txn.commit()?; Ok(()) diff --git a/http-ui/src/main.rs b/http-ui/src/main.rs index 8efd8ed69..4bd8815a5 100644 --- a/http-ui/src/main.rs +++ b/http-ui/src/main.rs @@ -343,7 +343,7 @@ async fn main() -> anyhow::Result<()> { // the type hint is necessary: https://github.com/rust-lang/rust/issues/32600 move |update_id, meta, content: &_| { // We prepare the update by using the update builder. - let mut update_builder = UpdateBuilder::new(update_id); + let mut update_builder = UpdateBuilder::new(); if let Some(max_nb_chunks) = indexer_opt_cloned.max_nb_chunks { update_builder.max_nb_chunks(max_nb_chunks); } @@ -393,7 +393,7 @@ async fn main() -> anyhow::Result<()> { let documents = DocumentBatchReader::from_reader(Cursor::new(documents))?; - let result = builder.execute(documents, |indexing_step, update_id| { + let result = builder.execute(documents, |indexing_step| { let (current, total) = match indexing_step { RemapDocumentAddition { documents_seen } => (documents_seen, None), ComputeIdsAndMergeDocuments { documents_seen, total_documents } => { @@ -494,7 +494,7 @@ async fn main() -> anyhow::Result<()> { Setting::NotSet => (), } - let result = builder.execute(|indexing_step, update_id| { + let result = builder.execute(|indexing_step| { let (current, total) = match indexing_step { RemapDocumentAddition { documents_seen } => (documents_seen, None), ComputeIdsAndMergeDocuments { documents_seen, total_documents } => { diff --git a/milli/src/index.rs b/milli/src/index.rs index fe89fe734..2f51b8c6b 100644 --- a/milli/src/index.rs +++ b/milli/src/index.rs @@ -908,8 +908,8 @@ pub(crate) mod tests { { "id": 2, "name": "bob", "age": 20 }, { "id": 2, "name": "bob", "age": 20 } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); @@ -927,13 +927,13 @@ pub(crate) mod tests { // we add all the documents a second time. we are supposed to get the same // field_distribution in the end let mut wtxn = index.write_txn().unwrap(); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); + let builder = IndexDocuments::new(&mut wtxn, &index); let content = documents!([ { "id": 1, "name": "kevin" }, { "id": 2, "name": "bob", "age": 20 }, { "id": 2, "name": "bob", "age": 20 } ]); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); @@ -955,8 +955,8 @@ pub(crate) mod tests { ]); let mut wtxn = index.write_txn().unwrap(); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); diff --git a/milli/src/search/distinct/mod.rs b/milli/src/search/distinct/mod.rs index 11f6379e3..3d36ed2a3 100644 --- a/milli/src/search/distinct/mod.rs +++ b/milli/src/search/distinct/mod.rs @@ -84,19 +84,19 @@ mod test { let mut txn = index.write_txn().unwrap(); // set distinct and faceted attributes for the index. - let builder = UpdateBuilder::new(0); + let builder = UpdateBuilder::new(); let mut update = builder.settings(&mut txn, &index); update.set_distinct_field(distinct.to_string()); - update.execute(|_, _| ()).unwrap(); + update.execute(|_| ()).unwrap(); // add documents to the index - let builder = UpdateBuilder::new(1); + let builder = UpdateBuilder::new(); let mut addition = builder.index_documents(&mut txn, &index); addition.index_documents_method(IndexDocumentsMethod::ReplaceDocuments); let reader = crate::documents::DocumentBatchReader::from_reader(Cursor::new(&*JSON)).unwrap(); - addition.execute(reader, |_, _| ()).unwrap(); + addition.execute(reader, |_| ()).unwrap(); let fields_map = index.fields_ids_map(&txn).unwrap(); let fid = fields_map.id(&distinct).unwrap(); diff --git a/milli/src/search/facet/filter.rs b/milli/src/search/facet/filter.rs index ec2c0b3eb..e994f36d9 100644 --- a/milli/src/search/facet/filter.rs +++ b/milli/src/search/facet/filter.rs @@ -512,10 +512,10 @@ mod tests { // Set the filterable fields to be the channel. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_searchable_fields(vec![S("title")]); builder.set_filterable_fields(hashset! { S("title") }); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); @@ -542,10 +542,10 @@ mod tests { // Set the filterable fields to be the channel. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_searchable_fields(vec![S("_geo"), S("price")]); // to keep the fields order builder.set_filterable_fields(hashset! { S("_geo"), S("price") }); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); diff --git a/milli/src/update/clear_documents.rs b/milli/src/update/clear_documents.rs index a820c2a49..5be3bc23d 100644 --- a/milli/src/update/clear_documents.rs +++ b/milli/src/update/clear_documents.rs @@ -6,16 +6,11 @@ use crate::{ExternalDocumentsIds, FieldDistribution, Index, Result}; pub struct ClearDocuments<'t, 'u, 'i> { wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, - _update_id: u64, } impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> { - pub fn new( - wtxn: &'t mut heed::RwTxn<'i, 'u>, - index: &'i Index, - update_id: u64, - ) -> ClearDocuments<'t, 'u, 'i> { - ClearDocuments { wtxn, index, _update_id: update_id } + pub fn new(wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index) -> ClearDocuments<'t, 'u, 'i> { + ClearDocuments { wtxn, index } } pub fn execute(self) -> Result { @@ -97,10 +92,10 @@ mod tests { { "id": 1, "name": "kevina" }, { "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } } ]); - IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap(); + IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap(); // Clear all documents from the database. - let builder = ClearDocuments::new(&mut wtxn, &index, 1); + let builder = ClearDocuments::new(&mut wtxn, &index); assert_eq!(builder.execute().unwrap(), 3); wtxn.commit().unwrap(); diff --git a/milli/src/update/delete_documents.rs b/milli/src/update/delete_documents.rs index d9c3fba14..2fd3e084e 100644 --- a/milli/src/update/delete_documents.rs +++ b/milli/src/update/delete_documents.rs @@ -23,7 +23,6 @@ pub struct DeleteDocuments<'t, 'u, 'i> { index: &'i Index, external_documents_ids: ExternalDocumentsIds<'static>, documents_ids: RoaringBitmap, - update_id: u64, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -36,7 +35,6 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { pub fn new( wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, - update_id: u64, ) -> Result> { let external_documents_ids = index.external_documents_ids(wtxn)?.into_static(); @@ -45,7 +43,6 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { index, external_documents_ids, documents_ids: RoaringBitmap::new(), - update_id, }) } @@ -85,8 +82,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> { // We can execute a ClearDocuments operation when the number of documents // to delete is exactly the number of documents in the database. if current_documents_ids_len == self.documents_ids.len() { - let remaining_documents = - ClearDocuments::new(self.wtxn, self.index, self.update_id).execute()?; + let remaining_documents = ClearDocuments::new(self.wtxn, self.index).execute()?; return Ok(DocumentDeletionResult { deleted_documents: current_documents_ids_len, remaining_documents, @@ -600,11 +596,11 @@ mod tests { { "id": 1, "name": "kevina", "array": ["I", "am", "fine"] }, { "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); // delete those documents, ids are synchronous therefore 0, 1, and 2. - let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap(); + let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); builder.delete_document(0); builder.delete_document(1); builder.delete_document(2); @@ -630,11 +626,11 @@ mod tests { { "mysuperid": 1, "name": "kevina" }, { "mysuperid": 2, "name": "benoit" } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); // Delete not all of the documents but some of them. - let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap(); + let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); builder.delete_external_id("0"); builder.delete_external_id("1"); builder.execute().unwrap(); @@ -650,10 +646,10 @@ mod tests { let index = Index::new(options, &path).unwrap(); let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_primary_key(S("docid")); builder.set_filterable_fields(hashset! { S("label") }); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); let content = documents!([ {"docid":"1_4","label":"sign"}, @@ -677,11 +673,11 @@ mod tests { {"docid":"1_68","label":"design"}, {"docid":"1_69","label":"geometry"} ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); // Delete not all of the documents but some of them. - let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap(); + let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); builder.delete_external_id("1_4"); builder.execute().unwrap(); @@ -700,11 +696,11 @@ mod tests { let index = Index::new(options, &path).unwrap(); let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_primary_key(S("id")); builder.set_filterable_fields(hashset!(S("_geo"))); builder.set_sortable_fields(hashset!(S("_geo"))); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); let content = documents!([ {"id":"1","city":"Lille", "_geo": { "lat": 50.629973371633746, "lng": 3.0569447399419570 } }, @@ -730,7 +726,7 @@ mod tests { ]); let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"]; - IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap(); + IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap(); let external_document_ids = index.external_documents_ids(&wtxn).unwrap(); let ids_to_delete: Vec = external_ids_to_delete @@ -739,7 +735,7 @@ mod tests { .collect(); // Delete some documents. - let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap(); + let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); external_ids_to_delete.iter().for_each(|id| drop(builder.delete_external_id(id))); builder.execute().unwrap(); diff --git a/milli/src/update/facets.rs b/milli/src/update/facets.rs index 9b7d6d42c..a2f17cba3 100644 --- a/milli/src/update/facets.rs +++ b/milli/src/update/facets.rs @@ -27,15 +27,10 @@ pub struct Facets<'t, 'u, 'i> { pub(crate) chunk_compression_level: Option, level_group_size: NonZeroUsize, min_level_size: NonZeroUsize, - _update_id: u64, } impl<'t, 'u, 'i> Facets<'t, 'u, 'i> { - pub fn new( - wtxn: &'t mut heed::RwTxn<'i, 'u>, - index: &'i Index, - update_id: u64, - ) -> Facets<'t, 'u, 'i> { + pub fn new(wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index) -> Facets<'t, 'u, 'i> { Facets { wtxn, index, @@ -43,7 +38,6 @@ impl<'t, 'u, 'i> Facets<'t, 'u, 'i> { chunk_compression_level: None, level_group_size: NonZeroUsize::new(4).unwrap(), min_level_size: NonZeroUsize::new(5).unwrap(), - _update_id: update_id, } } diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index cb3c1a75c..b0c0a5362 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -80,14 +80,12 @@ pub struct IndexDocuments<'t, 'u, 'i, 'a> { words_positions_min_level_size: Option, update_method: IndexDocumentsMethod, autogenerate_docids: bool, - update_id: u64, } impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { pub fn new( wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, - update_id: u64, ) -> IndexDocuments<'t, 'u, 'i, 'a> { IndexDocuments { wtxn, @@ -107,7 +105,6 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { words_positions_min_level_size: None, update_method: IndexDocumentsMethod::ReplaceDocuments, autogenerate_docids: false, - update_id, max_positions_per_attributes: None, } } @@ -136,7 +133,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { ) -> Result where R: Read + Seek, - F: Fn(UpdateIndexingStep, u64) + Sync, + F: Fn(UpdateIndexingStep) + Sync, { // Early return when there is no document to add if reader.is_empty() { @@ -148,8 +145,6 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { self.index.set_updated_at(self.wtxn, &Utc::now())?; let before_transform = Instant::now(); - let update_id = self.update_id; - let progress_callback = |step| progress_callback(step, update_id); let transform = Transform { rtxn: &self.wtxn, index: self.index, @@ -162,7 +157,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { autogenerate_docids: self.autogenerate_docids, }; - let output = transform.read_documents(reader, progress_callback)?; + let output = transform.read_documents(reader, &progress_callback)?; let indexed_documents = output.documents_count as u64; info!("Update transformed in {:.02?}", before_transform.elapsed()); @@ -295,7 +290,6 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { chunk_compression_level: self.chunk_compression_level, thread_pool: self.thread_pool, max_positions_per_attributes: self.max_positions_per_attributes, - update_id: self.update_id, }; let mut deletion_builder = update_builder.delete_documents(self.wtxn, self.index)?; debug!("documents to delete {:?}", replaced_documents_ids); @@ -364,7 +358,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { let mut databases_seen = MERGED_DATABASE_COUNT; // Run the facets update operation. - let mut builder = Facets::new(self.wtxn, self.index, self.update_id); + let mut builder = Facets::new(self.wtxn, self.index); builder.chunk_compression_type = self.chunk_compression_type; builder.chunk_compression_level = self.chunk_compression_level; if let Some(value) = self.facet_level_group_size { @@ -382,7 +376,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> { }); // Run the words prefixes update operation. - let mut builder = WordsPrefixesFst::new(self.wtxn, self.index, self.update_id); + let mut builder = WordsPrefixesFst::new(self.wtxn, self.index); if let Some(value) = self.words_prefix_threshold { builder.threshold(value); } @@ -475,8 +469,8 @@ mod tests { { "id": 2, "name": "kevina" }, { "id": 3, "name": "benoit" } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is 3 documents now. @@ -488,8 +482,8 @@ mod tests { // Second we send 1 document with id 1, to erase the previous ones. let mut wtxn = index.write_txn().unwrap(); let content = documents!([ { "id": 1, "name": "updated kevin" } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 1); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is **always** 3 documents. @@ -505,8 +499,8 @@ mod tests { { "id": 2, "name": "updated kevina" }, { "id": 3, "name": "updated benoit" } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 2); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is **always** 3 documents. @@ -531,9 +525,9 @@ mod tests { { "id": 1, "name": "kevina" }, { "id": 1, "name": "benoit" } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.index_documents_method(IndexDocumentsMethod::UpdateDocuments); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is only 1 document now. @@ -557,9 +551,9 @@ mod tests { // Second we send 1 document with id 1, to force it to be merged with the previous one. let mut wtxn = index.write_txn().unwrap(); let content = documents!([ { "id": 1, "age": 25 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 1); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.index_documents_method(IndexDocumentsMethod::UpdateDocuments); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is **always** 1 document. @@ -596,8 +590,8 @@ mod tests { { "name": "kevina" }, { "name": "benoit" } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - assert!(builder.execute(content, |_, _| ()).is_err()); + let builder = IndexDocuments::new(&mut wtxn, &index); + assert!(builder.execute(content, |_| ()).is_err()); wtxn.commit().unwrap(); // Check that there is no document. @@ -621,9 +615,9 @@ mod tests { { "name": "kevina" }, { "name": "benoit" } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is 3 documents now. @@ -639,8 +633,8 @@ mod tests { // Second we send 1 document with the generated uuid, to erase the previous ones. let mut wtxn = index.write_txn().unwrap(); let content = documents!([ { "name": "updated kevin", "id": kevin_uuid } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 1); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is **always** 3 documents. @@ -676,8 +670,8 @@ mod tests { { "id": 2, "name": "kevina" }, { "id": 3, "name": "benoit" } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is 3 documents now. @@ -689,9 +683,9 @@ mod tests { // Second we send 1 document without specifying the id. let mut wtxn = index.write_txn().unwrap(); let content = documents!([ { "name": "new kevin" } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 1); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is 4 documents now. @@ -711,8 +705,8 @@ mod tests { // First we send 0 documents and only headers. let mut wtxn = index.write_txn().unwrap(); let content = documents!([]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is no documents. @@ -733,16 +727,16 @@ mod tests { let mut wtxn = index.write_txn().unwrap(); // There is a space in the document id. let content = documents!([ { "id": "brume bleue", "name": "kevin" } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - assert!(builder.execute(content, |_, _| ()).is_err()); + let builder = IndexDocuments::new(&mut wtxn, &index); + assert!(builder.execute(content, |_| ()).is_err()); wtxn.commit().unwrap(); // First we send 1 document with a valid id. let mut wtxn = index.write_txn().unwrap(); // There is a space in the document id. let content = documents!([ { "id": 32, "name": "kevin" } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 1); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is 1 document now. @@ -766,8 +760,8 @@ mod tests { { "id": 1, "name": "kevina", "array": ["I", "am", "fine"] }, { "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is 1 documents now. @@ -805,13 +799,13 @@ mod tests { { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "_geo": { "lat": 35, "lng": 23 } } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.index_documents_method(IndexDocumentsMethod::ReplaceDocuments); - builder.execute(documents, |_, _| ()).unwrap(); + builder.execute(documents, |_| ()).unwrap(); wtxn.commit().unwrap(); let mut wtxn = index.write_txn().unwrap(); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 1); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.index_documents_method(IndexDocumentsMethod::UpdateDocuments); let documents = documents!([ { @@ -821,7 +815,7 @@ mod tests { } ]); - builder.execute(documents, |_, _| ()).unwrap(); + builder.execute(documents, |_| ()).unwrap(); wtxn.commit().unwrap(); } @@ -839,12 +833,12 @@ mod tests { { "objectId": 1, "title": "Alice In Wonderland", "comment": "A weird book" }, { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } ]); - IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap(); + IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap(); assert_eq!(index.primary_key(&wtxn).unwrap(), Some("objectId")); // Delete not all of the documents but some of them. - let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap(); + let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); builder.delete_external_id("30"); builder.execute().unwrap(); @@ -854,7 +848,7 @@ mod tests { let content = documents!([ { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } ]); - IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap(); + IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap(); let external_documents_ids = index.external_documents_ids(&wtxn).unwrap(); assert!(external_documents_ids.get("30").is_some()); @@ -862,7 +856,7 @@ mod tests { let content = documents!([ { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } ]); - IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap(); + IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); } @@ -892,8 +886,8 @@ mod tests { cursor.set_position(0); let content = DocumentBatchReader::from_reader(cursor).unwrap(); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); } @@ -922,8 +916,8 @@ mod tests { cursor.set_position(0); let content = DocumentBatchReader::from_reader(cursor).unwrap(); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); @@ -975,8 +969,8 @@ mod tests { }, ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); } @@ -996,8 +990,8 @@ mod tests { ]); let mut wtxn = index.write_txn().unwrap(); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is 4 document now. @@ -1008,8 +1002,8 @@ mod tests { let content = documents!([]); let mut wtxn = index.write_txn().unwrap(); - let builder = IndexDocuments::new(&mut wtxn, &index, 1); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is 4 document now. @@ -1025,8 +1019,8 @@ mod tests { ]); let mut wtxn = index.write_txn().unwrap(); - let builder = IndexDocuments::new(&mut wtxn, &index, 2); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that there is 4 document now. @@ -1048,8 +1042,8 @@ mod tests { ]); let mut wtxn = index.write_txn().unwrap(); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); diff --git a/milli/src/update/settings.rs b/milli/src/update/settings.rs index 0326b5144..9c270ed71 100644 --- a/milli/src/update/settings.rs +++ b/milli/src/update/settings.rs @@ -85,7 +85,6 @@ pub struct Settings<'a, 't, 'u, 'i> { pub(crate) chunk_compression_level: Option, pub(crate) thread_pool: Option<&'a ThreadPool>, pub(crate) max_positions_per_attributes: Option, - update_id: u64, searchable_fields: Setting>, displayed_fields: Setting>, @@ -99,11 +98,7 @@ pub struct Settings<'a, 't, 'u, 'i> { } impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { - pub fn new( - wtxn: &'t mut heed::RwTxn<'i, 'u>, - index: &'i Index, - update_id: u64, - ) -> Settings<'a, 't, 'u, 'i> { + pub fn new(wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index) -> Settings<'a, 't, 'u, 'i> { Settings { wtxn, index, @@ -123,7 +118,6 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { distinct_field: Setting::NotSet, synonyms: Setting::NotSet, primary_key: Setting::NotSet, - update_id, max_positions_per_attributes: None, } } @@ -207,11 +201,9 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { fn reindex(&mut self, cb: &F, old_fields_ids_map: FieldsIdsMap) -> Result<()> where - F: Fn(UpdateIndexingStep, u64) + Sync, + F: Fn(UpdateIndexingStep) + Sync, { let fields_ids_map = self.index.fields_ids_map(self.wtxn)?; - let update_id = self.update_id; - let cb = |step| cb(step, update_id); // if the settings are set before any document update, we don't need to do anything, and // will set the primary key during the first document addition. if self.index.number_of_documents(&self.wtxn)? == 0 { @@ -242,11 +234,11 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { )?; // We clear the full database (words-fst, documents ids and documents content). - ClearDocuments::new(self.wtxn, self.index, self.update_id).execute()?; + ClearDocuments::new(self.wtxn, self.index).execute()?; // We index the generated `TransformOutput` which must contain // all the documents with fields in the newly defined searchable order. - let mut indexing_builder = IndexDocuments::new(self.wtxn, self.index, self.update_id); + let mut indexing_builder = IndexDocuments::new(self.wtxn, self.index); indexing_builder.log_every_n = self.log_every_n; indexing_builder.max_nb_chunks = self.max_nb_chunks; indexing_builder.max_memory = self.max_memory; @@ -484,7 +476,7 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { pub fn execute(mut self, progress_callback: F) -> Result<()> where - F: Fn(UpdateIndexingStep, u64) + Sync, + F: Fn(UpdateIndexingStep) + Sync, { self.index.set_updated_at(self.wtxn, &Utc::now())?; @@ -543,15 +535,15 @@ mod tests { { "id": 2, "name": "kevina", "age": 21}, { "id": 3, "name": "benoit", "age": 34 } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 0); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // We change the searchable fields to be the "name" field only. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 1); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_searchable_fields(vec!["name".into()]); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // Check that the searchable field is correctly set to "name" only. @@ -571,9 +563,9 @@ mod tests { // We change the searchable fields to be the "name" field only. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 2); + let mut builder = Settings::new(&mut wtxn, &index); builder.reset_searchable_fields(); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // Check that the searchable field have been reset and documents are found now. @@ -600,18 +592,18 @@ mod tests { { "name": "kevina", "age": 21 }, { "name": "benoit", "age": 34 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // In the same transaction we change the displayed fields to be only the "age". // We also change the searchable fields to be the "name" field only. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 1); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_displayed_fields(vec!["age".into()]); builder.set_searchable_fields(vec!["name".into()]); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // Check that the displayed fields are correctly set to `None` (default value). @@ -622,9 +614,9 @@ mod tests { // We change the searchable fields to be the "name" field only. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 2); + let mut builder = Settings::new(&mut wtxn, &index); builder.reset_searchable_fields(); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // Check that the displayed fields always contains only the "age" field. @@ -647,9 +639,9 @@ mod tests { { "name": "kevina", "age": 21 }, { "name": "benoit", "age": 34 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that the displayed fields are correctly set to `None` (default value). @@ -672,14 +664,14 @@ mod tests { { "name": "kevina", "age": 21 }, { "name": "benoit", "age": 34 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); // In the same transaction we change the displayed fields to be only the age. - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_displayed_fields(vec!["age".into()]); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // Check that the displayed fields are correctly set to only the "age" field. @@ -690,9 +682,9 @@ mod tests { // We reset the fields ids to become `None`, the default value. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.reset_displayed_fields(); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // Check that the displayed fields are correctly set to `None` (default value). @@ -710,9 +702,9 @@ mod tests { // Set the filterable fields to be the age. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_filterable_fields(hashset! { S("age") }); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); // Then index some documents. let content = documents!([ @@ -720,9 +712,9 @@ mod tests { { "name": "kevina", "age": 21 }, { "name": "benoit", "age": 34 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 1); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Check that the displayed fields are correctly set. @@ -757,9 +749,9 @@ mod tests { { "name": "benoit", "age": 35 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 2); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); @@ -782,11 +774,11 @@ mod tests { // Set the filterable fields to be the age. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); // Don't display the generated `id` field. builder.set_displayed_fields(vec![S("name")]); builder.set_criteria(vec![S("age:asc")]); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); // Then index some documents. let content = documents!([ @@ -794,9 +786,9 @@ mod tests { { "name": "kevina", "age": 21 }, { "name": "benoit", "age": 34 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 1); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Run an empty query just to ensure that the search results are ordered. @@ -824,11 +816,11 @@ mod tests { // Set the filterable fields to be the age. let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); // Don't display the generated `id` field. builder.set_displayed_fields(vec![S("name"), S("age")]); builder.set_distinct_field(S("age")); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); // Then index some documents. let content = documents!([ @@ -840,9 +832,9 @@ mod tests { { "name": "bernie", "age": 34 }, { "name": "ben", "age": 34 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 1); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Run an empty query just to ensure that the search results are ordered. @@ -867,9 +859,9 @@ mod tests { { "name": "kevina", "age": 21 }, { "name": "benoit", "age": 34 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // Ensure there is no stop_words by default @@ -892,15 +884,15 @@ mod tests { { "name": "kevina", "age": 21, "maxim": "Doggos are the best" }, { "name": "benoit", "age": 34, "maxim": "The crepes are really good" }, ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); // In the same transaction we provide some stop_words - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); let set = btreeset! { "i".to_string(), "the".to_string(), "are".to_string() }; builder.set_stop_words(set.clone()); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // Ensure stop_words are effectively stored @@ -928,9 +920,9 @@ mod tests { // now we'll reset the stop_words and ensure it's None let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.reset_stop_words(); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); @@ -966,18 +958,18 @@ mod tests { { "name": "kevina", "age": 21, "maxim": "Doggos are the best"}, { "name": "benoit", "age": 34, "maxim": "The crepes are really good"}, ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.enable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); // In the same transaction provide some synonyms - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_synonyms(hashmap! { "blini".to_string() => vec!["crepes".to_string()], "super like".to_string() => vec!["love".to_string()], "puppies".to_string() => vec!["dogs".to_string(), "doggos".to_string()] }); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // Ensure synonyms are effectively stored @@ -995,9 +987,9 @@ mod tests { // Reset the synonyms let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.reset_synonyms(); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // Ensure synonyms are reset @@ -1023,11 +1015,11 @@ mod tests { // Set all the settings except searchable let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_displayed_fields(vec!["hello".to_string()]); builder.set_filterable_fields(hashset! { S("age"), S("toto") }); builder.set_criteria(vec!["toto:asc".to_string()]); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); // check the output @@ -1040,9 +1032,9 @@ mod tests { // We set toto and age as searchable to force reordering of the fields let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 1); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_searchable_fields(vec!["toto".to_string(), "age".to_string()]); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1060,11 +1052,11 @@ mod tests { // Set all the settings except searchable let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_displayed_fields(vec!["hello".to_string()]); // It is only Asc(toto), there is a facet database but it is denied to filter with toto. builder.set_criteria(vec!["toto:asc".to_string()]); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1081,10 +1073,10 @@ mod tests { // Set the primary key settings let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_primary_key(S("mykey")); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); assert_eq!(index.primary_key(&wtxn).unwrap(), Some("mykey")); // Then index some documents with the "mykey" primary key. @@ -1097,29 +1089,29 @@ mod tests { { "mykey": 6, "name": "bernie", "age": 34 }, { "mykey": 7, "name": "ben", "age": 34 } ]); - let mut builder = IndexDocuments::new(&mut wtxn, &index, 1); + let mut builder = IndexDocuments::new(&mut wtxn, &index); builder.disable_autogenerate_docids(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // We now try to reset the primary key let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.reset_primary_key(); - let err = builder.execute(|_, _| ()).unwrap_err(); + let err = builder.execute(|_| ()).unwrap_err(); assert!(matches!(err, Error::UserError(UserError::PrimaryKeyCannotBeChanged(_)))); wtxn.abort().unwrap(); // But if we clear the database... let mut wtxn = index.write_txn().unwrap(); - let builder = ClearDocuments::new(&mut wtxn, &index, 0); + let builder = ClearDocuments::new(&mut wtxn, &index); builder.execute().unwrap(); // ...we can change the primary key - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_primary_key(S("myid")); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); } @@ -1132,9 +1124,9 @@ mod tests { // Set the genres setting let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_filterable_fields(hashset! { S("genres") }); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); let content = documents!([ { @@ -1155,8 +1147,8 @@ mod tests { "release_date": 819676800 } ]); - let builder = IndexDocuments::new(&mut wtxn, &index, 1); - builder.execute(content, |_, _| ()).unwrap(); + let builder = IndexDocuments::new(&mut wtxn, &index); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); // We now try to reset the primary key diff --git a/milli/src/update/update_builder.rs b/milli/src/update/update_builder.rs index 20ec28e06..6e892a356 100644 --- a/milli/src/update/update_builder.rs +++ b/milli/src/update/update_builder.rs @@ -13,11 +13,10 @@ pub struct UpdateBuilder<'a> { pub(crate) chunk_compression_level: Option, pub(crate) thread_pool: Option<&'a ThreadPool>, pub(crate) max_positions_per_attributes: Option, - pub(crate) update_id: u64, } impl<'a> UpdateBuilder<'a> { - pub fn new(update_id: u64) -> UpdateBuilder<'a> { + pub fn new() -> UpdateBuilder<'a> { UpdateBuilder { log_every_n: None, max_nb_chunks: None, @@ -27,7 +26,6 @@ impl<'a> UpdateBuilder<'a> { chunk_compression_level: None, thread_pool: None, max_positions_per_attributes: None, - update_id, } } @@ -68,7 +66,7 @@ impl<'a> UpdateBuilder<'a> { wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, ) -> ClearDocuments<'t, 'u, 'i> { - ClearDocuments::new(wtxn, index, self.update_id) + ClearDocuments::new(wtxn, index) } pub fn delete_documents<'t, 'u, 'i>( @@ -76,7 +74,7 @@ impl<'a> UpdateBuilder<'a> { wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, ) -> Result> { - DeleteDocuments::new(wtxn, index, self.update_id) + DeleteDocuments::new(wtxn, index) } pub fn index_documents<'t, 'u, 'i>( @@ -84,7 +82,7 @@ impl<'a> UpdateBuilder<'a> { wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, ) -> IndexDocuments<'t, 'u, 'i, 'a> { - let mut builder = IndexDocuments::new(wtxn, index, self.update_id); + let mut builder = IndexDocuments::new(wtxn, index); builder.log_every_n = self.log_every_n; builder.max_nb_chunks = self.max_nb_chunks; @@ -103,7 +101,7 @@ impl<'a> UpdateBuilder<'a> { wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, ) -> Settings<'a, 't, 'u, 'i> { - let mut builder = Settings::new(wtxn, index, self.update_id); + let mut builder = Settings::new(wtxn, index); builder.log_every_n = self.log_every_n; builder.max_nb_chunks = self.max_nb_chunks; @@ -122,7 +120,7 @@ impl<'a> UpdateBuilder<'a> { wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, ) -> Facets<'t, 'u, 'i> { - let mut builder = Facets::new(wtxn, index, self.update_id); + let mut builder = Facets::new(wtxn, index); builder.chunk_compression_type = self.chunk_compression_type; builder.chunk_compression_level = self.chunk_compression_level; diff --git a/milli/src/update/words_prefixes_fst.rs b/milli/src/update/words_prefixes_fst.rs index eaaacc26f..49406deb5 100644 --- a/milli/src/update/words_prefixes_fst.rs +++ b/milli/src/update/words_prefixes_fst.rs @@ -10,22 +10,14 @@ pub struct WordsPrefixesFst<'t, 'u, 'i> { index: &'i Index, threshold: u32, max_prefix_length: usize, - _update_id: u64, } impl<'t, 'u, 'i> WordsPrefixesFst<'t, 'u, 'i> { pub fn new( wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, - update_id: u64, ) -> WordsPrefixesFst<'t, 'u, 'i> { - WordsPrefixesFst { - wtxn, - index, - threshold: 100, - max_prefix_length: 4, - _update_id: update_id, - } + WordsPrefixesFst { wtxn, index, threshold: 100, max_prefix_length: 4 } } /// Set the number of words required to make a prefix be part of the words prefixes diff --git a/milli/tests/search/distinct.rs b/milli/tests/search/distinct.rs index f044756eb..da7251389 100644 --- a/milli/tests/search/distinct.rs +++ b/milli/tests/search/distinct.rs @@ -16,9 +16,9 @@ macro_rules! test_distinct { // update distinct attribute let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_distinct_field(S(stringify!($distinct))); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); diff --git a/milli/tests/search/mod.rs b/milli/tests/search/mod.rs index e8fb3fdfa..d1467fd72 100644 --- a/milli/tests/search/mod.rs +++ b/milli/tests/search/mod.rs @@ -32,7 +32,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); let criteria = criteria.iter().map(|c| c.to_string()).collect(); builder.set_criteria(criteria); @@ -51,10 +51,10 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { S("america") => vec![S("the united states")], }); builder.set_searchable_fields(vec![S("title"), S("description")]); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); // index documents - let mut builder = UpdateBuilder::new(0); + let mut builder = UpdateBuilder::new(); builder.max_memory(10 * 1024 * 1024); // 10MiB let mut builder = builder.index_documents(&mut wtxn, &index); builder.enable_autogenerate_docids(); @@ -73,7 +73,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { // index documents let content = DocumentBatchReader::from_reader(cursor).unwrap(); - builder.execute(content, |_, _| ()).unwrap(); + builder.execute(content, |_| ()).unwrap(); wtxn.commit().unwrap(); diff --git a/milli/tests/search/query_criteria.rs b/milli/tests/search/query_criteria.rs index e5dde049c..8968eff90 100644 --- a/milli/tests/search/query_criteria.rs +++ b/milli/tests/search/query_criteria.rs @@ -341,9 +341,9 @@ fn criteria_mixup() { eprintln!("Testing with criteria order: {:?}", &criteria); //update criteria let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_criteria(criteria.iter().map(ToString::to_string).collect()); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); let mut rtxn = index.read_txn().unwrap(); @@ -376,16 +376,16 @@ fn criteria_ascdesc() { let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_sortable_fields(hashset! { S("name"), S("age"), }); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); // index documents - let mut builder = UpdateBuilder::new(0); + let mut builder = UpdateBuilder::new(); builder.max_memory(10 * 1024 * 1024); // 10MiB let mut builder = builder.index_documents(&mut wtxn, &index); builder.enable_autogenerate_docids(); @@ -419,7 +419,7 @@ fn criteria_ascdesc() { let reader = DocumentBatchReader::from_reader(cursor).unwrap(); - builder.execute(reader, |_, _| ()).unwrap(); + builder.execute(reader, |_| ()).unwrap(); wtxn.commit().unwrap(); @@ -430,9 +430,9 @@ fn criteria_ascdesc() { eprintln!("Testing with criterion: {:?}", &criterion); let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, 0); + let mut builder = Settings::new(&mut wtxn, &index); builder.set_criteria(vec![criterion.to_string()]); - builder.execute(|_, _| ()).unwrap(); + builder.execute(|_| ()).unwrap(); wtxn.commit().unwrap(); let mut rtxn = index.read_txn().unwrap();