From acff17fb88fff6cc61ab3ae5392669b59afef262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Tue, 2 Aug 2022 15:13:06 +0200 Subject: [PATCH] Simplify indexing tests --- milli/src/documents/mod.rs | 11 + milli/src/index.rs | 229 ++-- milli/src/update/clear_documents.rs | 29 +- milli/src/update/delete_documents.rs | 376 +++---- milli/src/update/index_documents/mod.rs | 1328 +++++++---------------- milli/src/update/settings.rs | 823 ++++++-------- 6 files changed, 1070 insertions(+), 1726 deletions(-) diff --git a/milli/src/documents/mod.rs b/milli/src/documents/mod.rs index e766e29cf..c1580309a 100644 --- a/milli/src/documents/mod.rs +++ b/milli/src/documents/mod.rs @@ -179,6 +179,17 @@ macro_rules! documents { }}; } +#[cfg(test)] +pub fn documents_batch_reader_from_objects( + objects: impl IntoIterator, +) -> DocumentsBatchReader>> { + let mut builder = DocumentsBatchBuilder::new(Vec::new()); + for object in objects { + builder.append_json_object(&object).unwrap(); + } + DocumentsBatchReader::from_reader(std::io::Cursor::new(builder.into_inner().unwrap())).unwrap() +} + #[cfg(test)] mod test { use std::io::Cursor; diff --git a/milli/src/index.rs b/milli/src/index.rs index 6d95332fd..43888a177 100644 --- a/milli/src/index.rs +++ b/milli/src/index.rs @@ -1182,16 +1182,19 @@ pub(crate) mod tests { use std::ops::Deref; use big_s::S; - use heed::EnvOpenOptions; + use heed::{EnvOpenOptions, RwTxn}; use maplit::btreemap; use tempfile::TempDir; + use crate::documents::DocumentsBatchReader; use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS}; - use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig}; + use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings}; use crate::Index; pub(crate) struct TempIndex { - inner: Index, + pub inner: Index, + pub indexer_config: IndexerConfig, + pub index_documents_config: IndexDocumentsConfig, _tempdir: TempDir, } @@ -1204,43 +1207,88 @@ pub(crate) mod tests { } impl TempIndex { - /// Creates a temporary index, with a default `4096 * 100` size. This should be enough for - /// most tests. - pub fn new() -> Self { + /// Creates a temporary index + pub fn new_with_map_size(size: usize) -> Self { let mut options = EnvOpenOptions::new(); - options.map_size(100 * 4096); + options.map_size(size); let _tempdir = TempDir::new_in(".").unwrap(); let inner = Index::new(options, _tempdir.path()).unwrap(); - Self { inner, _tempdir } + let indexer_config = IndexerConfig::default(); + let index_documents_config = IndexDocumentsConfig::default(); + Self { inner, indexer_config, index_documents_config, _tempdir } + } + /// Creates a temporary index, with a default `4096 * 1000` size. This should be enough for + /// most tests. + pub fn new() -> Self { + Self::new_with_map_size(4096 * 1000) + } + pub fn add_documents_using_wtxn<'t, R>( + &'t self, + wtxn: &mut RwTxn<'t, '_>, + documents: DocumentsBatchReader, + ) -> Result<(), crate::error::Error> + where + R: std::io::Read + std::io::Seek, + { + let builder = IndexDocuments::new( + wtxn, + &self, + &self.indexer_config, + self.index_documents_config.clone(), + |_| (), + ) + .unwrap(); + let (builder, user_error) = builder.add_documents(documents).unwrap(); + user_error?; + builder.execute()?; + Ok(()) + } + pub fn add_documents( + &self, + documents: DocumentsBatchReader, + ) -> Result<(), crate::error::Error> + where + R: std::io::Read + std::io::Seek, + { + let mut wtxn = self.write_txn().unwrap(); + self.add_documents_using_wtxn(&mut wtxn, documents)?; + wtxn.commit().unwrap(); + Ok(()) + } + + pub fn update_settings( + &self, + update: impl Fn(&mut Settings), + ) -> Result<(), crate::error::Error> { + let mut wtxn = self.write_txn().unwrap(); + self.update_settings_using_wtxn(&mut wtxn, update)?; + wtxn.commit().unwrap(); + Ok(()) + } + pub fn update_settings_using_wtxn<'t>( + &'t self, + wtxn: &mut RwTxn<'t, '_>, + update: impl Fn(&mut Settings), + ) -> Result<(), crate::error::Error> { + let mut builder = update::Settings::new(wtxn, &self.inner, &self.indexer_config); + update(&mut builder); + builder.execute(drop)?; + Ok(()) } } #[test] fn initial_field_distribution() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 1, "name": "kevin" }, - { "id": 2, "name": "bob", "age": 20 }, - { "id": 2, "name": "bob", "age": 20 }, - ]); - - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + let index = TempIndex::new(); + index + .add_documents(documents!([ + { "id": 1, "name": "kevin" }, + { "id": 2, "name": "bob", "age": 20 }, + { "id": 2, "name": "bob", "age": 20 }, + ])) + .unwrap(); let rtxn = index.read_txn().unwrap(); - let field_distribution = index.field_distribution(&rtxn).unwrap(); assert_eq!( field_distribution, @@ -1253,19 +1301,13 @@ pub(crate) mod tests { // we add all the documents a second time. we are supposed to get the same // field_distribution in the end - let mut wtxn = index.write_txn().unwrap(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let content = documents!([ - { "id": 1, "name": "kevin" }, - { "id": 2, "name": "bob", "age": 20 }, - { "id": 2, "name": "bob", "age": 20 }, - ]); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 1, "name": "kevin" }, + { "id": 2, "name": "bob", "age": 20 }, + { "id": 2, "name": "bob", "age": 20 }, + ])) + .unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1280,19 +1322,12 @@ pub(crate) mod tests { ); // then we update a document by removing one field and another by adding one field - let content = documents!([ - { "id": 1, "name": "kevin", "has_dog": true }, - { "id": 2, "name": "bob" } - ]); - - let mut wtxn = index.write_txn().unwrap(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 1, "name": "kevin", "has_dog": true }, + { "id": 2, "name": "bob" } + ])) + .unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1341,35 +1376,19 @@ pub(crate) mod tests { #[test] fn add_documents_and_set_searchable_fields() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 1, "doggo": "kevin" }, - { "id": 2, "doggo": { "name": "bob", "age": 20 } }, - { "id": 3, "name": "jean", "age": 25 }, - ]); - - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); - - // set searchable fields - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - builder.set_searchable_fields(vec![S("doggo"), S("name")]); - - builder.execute(drop).unwrap(); - wtxn.commit().unwrap(); + let index = TempIndex::new(); + index + .add_documents(documents!([ + { "id": 1, "doggo": "kevin" }, + { "id": 2, "doggo": { "name": "bob", "age": 20 } }, + { "id": 3, "name": "jean", "age": 25 }, + ])) + .unwrap(); + index + .update_settings(|settings| { + settings.set_searchable_fields(vec![S("doggo"), S("name")]); + }) + .unwrap(); // ensure we get the right real searchable fields + user defined searchable fields let rtxn = index.read_txn().unwrap(); @@ -1383,19 +1402,13 @@ pub(crate) mod tests { #[test] fn set_searchable_fields_and_add_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let index = TempIndex::new(); - // set searchable fields - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - builder.set_searchable_fields(vec![S("doggo"), S("name")]); - - builder.execute(drop).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_searchable_fields(vec![S("doggo"), S("name")]); + }) + .unwrap(); // ensure we get the right real searchable fields + user defined searchable fields let rtxn = index.read_txn().unwrap(); @@ -1405,21 +1418,13 @@ pub(crate) mod tests { let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap(); assert_eq!(user_defined, &["doggo", "name"]); - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 1, "doggo": "kevin" }, - { "id": 2, "doggo": { "name": "bob", "age": 20 } }, - { "id": 3, "name": "jean", "age": 25 }, - ]); - - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 1, "doggo": "kevin" }, + { "id": 2, "doggo": { "name": "bob", "age": 20 } }, + { "id": 3, "name": "jean", "age": 25 }, + ])) + .unwrap(); // ensure we get the right real searchable fields + user defined searchable fields let rtxn = index.read_txn().unwrap(); diff --git a/milli/src/update/clear_documents.rs b/milli/src/update/clear_documents.rs index 388865d56..5b7dbc57c 100644 --- a/milli/src/update/clear_documents.rs +++ b/milli/src/update/clear_documents.rs @@ -82,36 +82,25 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> { #[cfg(test)] mod tests { - use heed::EnvOpenOptions; - use super::*; - use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig}; + use crate::index::tests::TempIndex; #[test] fn clear_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 0, "name": "kevin", "age": 20 }, - { "id": 1, "name": "kevina" }, - { "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } } - ]); - let indexing_config = IndexDocumentsConfig::default(); - let config = IndexerConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); + index + .add_documents_using_wtxn(&mut wtxn, documents!([ + { "id": 0, "name": "kevin", "age": 20 }, + { "id": 1, "name": "kevina" }, + { "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } } + ])) + .unwrap(); // Clear all documents from the database. let builder = ClearDocuments::new(&mut wtxn, &index); assert_eq!(builder.execute().unwrap(), 3); - wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); diff --git a/milli/src/update/delete_documents.rs b/milli/src/update/delete_documents.rs index f10829454..c981ee061 100644 --- a/milli/src/update/delete_documents.rs +++ b/milli/src/update/delete_documents.rs @@ -654,26 +654,13 @@ where #[cfg(test)] mod tests { use big_s::S; - use heed::{EnvOpenOptions, RwTxn}; + use heed::RwTxn; use maplit::hashset; use super::*; - use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings}; + use crate::index::tests::TempIndex; use crate::Filter; - fn insert_documents<'t, R: std::io::Read + std::io::Seek>( - wtxn: &mut RwTxn<'t, '_>, - index: &'t Index, - documents: crate::documents::DocumentsBatchReader, - ) { - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = IndexDocuments::new(wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - } - fn delete_documents<'t>( wtxn: &mut RwTxn<'t, '_>, index: &'t Index, @@ -695,24 +682,19 @@ mod tests { #[test] fn delete_documents_with_numbers_as_primary_key() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } }, - { "id": 1, "name": "kevina", "array": ["I", "am", "fine"] }, - { "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] } - ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } }, + { "id": 1, "name": "kevina", "array": ["I", "am", "fine"] }, + { "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] } + ]), + ) + .unwrap(); // delete those documents, ids are synchronous therefore 0, 1, and 2. let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); @@ -730,25 +712,19 @@ mod tests { #[test] fn delete_documents_with_strange_primary_key() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "mysuperid": 0, "name": "kevin" }, - { "mysuperid": 1, "name": "kevina" }, - { "mysuperid": 2, "name": "benoit" } - ]); - - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "mysuperid": 0, "name": "kevin" }, + { "mysuperid": 1, "name": "kevina" }, + { "mysuperid": 2, "name": "benoit" } + ]), + ) + .unwrap(); // Delete not all of the documents but some of them. let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); @@ -761,42 +737,45 @@ mod tests { #[test] fn filtered_placeholder_search_should_not_return_deleted_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key(S("docid")); - builder.set_filterable_fields(hashset! { S("label") }); - builder.execute(|_| ()).unwrap(); - let content = documents!([ - { "docid": "1_4", "label": "sign" }, - { "docid": "1_5", "label": "letter" }, - { "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, - { "docid": "1_36", "label": "drawing,painting,pattern" }, - { "docid": "1_37", "label": "art,drawing,outdoor" }, - { "docid": "1_38", "label": "aquarium,art,drawing" }, - { "docid": "1_39", "label": "abstract" }, - { "docid": "1_40", "label": "cartoon" }, - { "docid": "1_41", "label": "art,drawing" }, - { "docid": "1_42", "label": "art,pattern" }, - { "docid": "1_43", "label": "abstract,art,drawing,pattern" }, - { "docid": "1_44", "label": "drawing" }, - { "docid": "1_45", "label": "art" }, - { "docid": "1_46", "label": "abstract,colorfulness,pattern" }, - { "docid": "1_47", "label": "abstract,pattern" }, - { "docid": "1_52", "label": "abstract,cartoon" }, - { "docid": "1_57", "label": "abstract,drawing,pattern" }, - { "docid": "1_58", "label": "abstract,art,cartoon" }, - { "docid": "1_68", "label": "design" }, - { "docid": "1_69", "label": "geometry" } - ]); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_primary_key(S("docid")); + settings.set_filterable_fields(hashset! { S("label") }); + }) + .unwrap(); + + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "docid": "1_4", "label": "sign" }, + { "docid": "1_5", "label": "letter" }, + { "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, + { "docid": "1_36", "label": "drawing,painting,pattern" }, + { "docid": "1_37", "label": "art,drawing,outdoor" }, + { "docid": "1_38", "label": "aquarium,art,drawing" }, + { "docid": "1_39", "label": "abstract" }, + { "docid": "1_40", "label": "cartoon" }, + { "docid": "1_41", "label": "art,drawing" }, + { "docid": "1_42", "label": "art,pattern" }, + { "docid": "1_43", "label": "abstract,art,drawing,pattern" }, + { "docid": "1_44", "label": "drawing" }, + { "docid": "1_45", "label": "art" }, + { "docid": "1_46", "label": "abstract,colorfulness,pattern" }, + { "docid": "1_47", "label": "abstract,pattern" }, + { "docid": "1_52", "label": "abstract,cartoon" }, + { "docid": "1_57", "label": "abstract,drawing,pattern" }, + { "docid": "1_58", "label": "abstract,art,cartoon" }, + { "docid": "1_68", "label": "design" }, + { "docid": "1_69", "label": "geometry" } + ]), + ) + .unwrap(); - insert_documents(&mut wtxn, &index, content); delete_documents(&mut wtxn, &index, &["1_4"]); // Placeholder search with filter @@ -809,41 +788,43 @@ mod tests { #[test] fn placeholder_search_should_not_return_deleted_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key(S("docid")); - builder.execute(|_| ()).unwrap(); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_primary_key(S("docid")); + }) + .unwrap(); - let content = documents!([ - { "docid": "1_4", "label": "sign" }, - { "docid": "1_5", "label": "letter" }, - { "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, - { "docid": "1_36", "label": "drawing,painting,pattern" }, - { "docid": "1_37", "label": "art,drawing,outdoor" }, - { "docid": "1_38", "label": "aquarium,art,drawing" }, - { "docid": "1_39", "label": "abstract" }, - { "docid": "1_40", "label": "cartoon" }, - { "docid": "1_41", "label": "art,drawing" }, - { "docid": "1_42", "label": "art,pattern" }, - { "docid": "1_43", "label": "abstract,art,drawing,pattern" }, - { "docid": "1_44", "label": "drawing" }, - { "docid": "1_45", "label": "art" }, - { "docid": "1_46", "label": "abstract,colorfulness,pattern" }, - { "docid": "1_47", "label": "abstract,pattern" }, - { "docid": "1_52", "label": "abstract,cartoon" }, - { "docid": "1_57", "label": "abstract,drawing,pattern" }, - { "docid": "1_58", "label": "abstract,art,cartoon" }, - { "docid": "1_68", "label": "design" }, - { "docid": "1_69", "label": "geometry" } - ]); + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "docid": "1_4", "label": "sign" }, + { "docid": "1_5", "label": "letter" }, + { "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, + { "docid": "1_36", "label": "drawing,painting,pattern" }, + { "docid": "1_37", "label": "art,drawing,outdoor" }, + { "docid": "1_38", "label": "aquarium,art,drawing" }, + { "docid": "1_39", "label": "abstract" }, + { "docid": "1_40", "label": "cartoon" }, + { "docid": "1_41", "label": "art,drawing" }, + { "docid": "1_42", "label": "art,pattern" }, + { "docid": "1_43", "label": "abstract,art,drawing,pattern" }, + { "docid": "1_44", "label": "drawing" }, + { "docid": "1_45", "label": "art" }, + { "docid": "1_46", "label": "abstract,colorfulness,pattern" }, + { "docid": "1_47", "label": "abstract,pattern" }, + { "docid": "1_52", "label": "abstract,cartoon" }, + { "docid": "1_57", "label": "abstract,drawing,pattern" }, + { "docid": "1_58", "label": "abstract,art,cartoon" }, + { "docid": "1_68", "label": "design" }, + { "docid": "1_69", "label": "geometry" } + ]), + ) + .unwrap(); - insert_documents(&mut wtxn, &index, content); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_4"]); // Placeholder search @@ -862,41 +843,43 @@ mod tests { #[test] fn search_should_not_return_deleted_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key(S("docid")); - builder.execute(|_| ()).unwrap(); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_primary_key(S("docid")); + }) + .unwrap(); - let content = documents!([ - {"docid": "1_4", "label": "sign"}, - {"docid": "1_5", "label": "letter"}, - {"docid": "1_7", "label": "abstract,cartoon,design,pattern"}, - {"docid": "1_36","label": "drawing,painting,pattern"}, - {"docid": "1_37","label": "art,drawing,outdoor"}, - {"docid": "1_38","label": "aquarium,art,drawing"}, - {"docid": "1_39","label": "abstract"}, - {"docid": "1_40","label": "cartoon"}, - {"docid": "1_41","label": "art,drawing"}, - {"docid": "1_42","label": "art,pattern"}, - {"docid": "1_43","label": "abstract,art,drawing,pattern"}, - {"docid": "1_44","label": "drawing"}, - {"docid": "1_45","label": "art"}, - {"docid": "1_46","label": "abstract,colorfulness,pattern"}, - {"docid": "1_47","label": "abstract,pattern"}, - {"docid": "1_52","label": "abstract,cartoon"}, - {"docid": "1_57","label": "abstract,drawing,pattern"}, - {"docid": "1_58","label": "abstract,art,cartoon"}, - {"docid": "1_68","label": "design"}, - {"docid": "1_69","label": "geometry"} - ]); + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + {"docid": "1_4", "label": "sign"}, + {"docid": "1_5", "label": "letter"}, + {"docid": "1_7", "label": "abstract,cartoon,design,pattern"}, + {"docid": "1_36","label": "drawing,painting,pattern"}, + {"docid": "1_37","label": "art,drawing,outdoor"}, + {"docid": "1_38","label": "aquarium,art,drawing"}, + {"docid": "1_39","label": "abstract"}, + {"docid": "1_40","label": "cartoon"}, + {"docid": "1_41","label": "art,drawing"}, + {"docid": "1_42","label": "art,pattern"}, + {"docid": "1_43","label": "abstract,art,drawing,pattern"}, + {"docid": "1_44","label": "drawing"}, + {"docid": "1_45","label": "art"}, + {"docid": "1_46","label": "abstract,colorfulness,pattern"}, + {"docid": "1_47","label": "abstract,pattern"}, + {"docid": "1_52","label": "abstract,cartoon"}, + {"docid": "1_57","label": "abstract,drawing,pattern"}, + {"docid": "1_58","label": "abstract,art,cartoon"}, + {"docid": "1_68","label": "design"}, + {"docid": "1_69","label": "geometry"} + ]), + ) + .unwrap(); - insert_documents(&mut wtxn, &index, content); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_7", "1_52"]); // search for abstract @@ -915,20 +898,18 @@ mod tests { #[test] fn geo_filtered_placeholder_search_should_not_return_deleted_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key(S("id")); - builder.set_filterable_fields(hashset!(S("_geo"))); - builder.set_sortable_fields(hashset!(S("_geo"))); - builder.execute(|_| ()).unwrap(); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_primary_key(S("id")); + settings.set_filterable_fields(hashset!(S("_geo"))); + settings.set_sortable_fields(hashset!(S("_geo"))); + }) + .unwrap(); - let content = documents!([ + index.add_documents_using_wtxn(&mut wtxn, documents!([ { "id": "1", "city": "Lille", "_geo": { "lat": 50.6299, "lng": 3.0569 } }, { "id": "2", "city": "Mons-en-Barœul", "_geo": { "lat": 50.6415, "lng": 3.1106 } }, { "id": "3", "city": "Hellemmes", "_geo": { "lat": 50.6312, "lng": 3.1106 } }, @@ -949,10 +930,9 @@ mod tests { { "id": "18", "city": "Amiens", "_geo": { "lat": 49.9314, "lng": 2.2710 } }, { "id": "19", "city": "Compiègne", "_geo": { "lat": 49.4449, "lng": 2.7913 } }, { "id": "20", "city": "Paris", "_geo": { "lat": 48.9021, "lng": 2.3708 } } - ]); - let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"]; + ])).unwrap(); - insert_documents(&mut wtxn, &index, content); + let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"]; let deleted_internal_ids = delete_documents(&mut wtxn, &index, &external_ids_to_delete); // Placeholder search with geo filter @@ -972,41 +952,43 @@ mod tests { #[test] fn get_documents_should_not_return_deleted_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key(S("docid")); - builder.execute(|_| ()).unwrap(); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_primary_key(S("docid")); + }) + .unwrap(); - let content = documents!([ - { "docid": "1_4", "label": "sign" }, - { "docid": "1_5", "label": "letter" }, - { "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, - { "docid": "1_36", "label": "drawing,painting,pattern" }, - { "docid": "1_37", "label": "art,drawing,outdoor" }, - { "docid": "1_38", "label": "aquarium,art,drawing" }, - { "docid": "1_39", "label": "abstract" }, - { "docid": "1_40", "label": "cartoon" }, - { "docid": "1_41", "label": "art,drawing" }, - { "docid": "1_42", "label": "art,pattern" }, - { "docid": "1_43", "label": "abstract,art,drawing,pattern" }, - { "docid": "1_44", "label": "drawing" }, - { "docid": "1_45", "label": "art" }, - { "docid": "1_46", "label": "abstract,colorfulness,pattern" }, - { "docid": "1_47", "label": "abstract,pattern" }, - { "docid": "1_52", "label": "abstract,cartoon" }, - { "docid": "1_57", "label": "abstract,drawing,pattern" }, - { "docid": "1_58", "label": "abstract,art,cartoon" }, - { "docid": "1_68", "label": "design" }, - { "docid": "1_69", "label": "geometry" } - ]); + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "docid": "1_4", "label": "sign" }, + { "docid": "1_5", "label": "letter" }, + { "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, + { "docid": "1_36", "label": "drawing,painting,pattern" }, + { "docid": "1_37", "label": "art,drawing,outdoor" }, + { "docid": "1_38", "label": "aquarium,art,drawing" }, + { "docid": "1_39", "label": "abstract" }, + { "docid": "1_40", "label": "cartoon" }, + { "docid": "1_41", "label": "art,drawing" }, + { "docid": "1_42", "label": "art,pattern" }, + { "docid": "1_43", "label": "abstract,art,drawing,pattern" }, + { "docid": "1_44", "label": "drawing" }, + { "docid": "1_45", "label": "art" }, + { "docid": "1_46", "label": "abstract,colorfulness,pattern" }, + { "docid": "1_47", "label": "abstract,pattern" }, + { "docid": "1_52", "label": "abstract,cartoon" }, + { "docid": "1_57", "label": "abstract,drawing,pattern" }, + { "docid": "1_58", "label": "abstract,art,cartoon" }, + { "docid": "1_68", "label": "design" }, + { "docid": "1_69", "label": "geometry" } + ]), + ) + .unwrap(); - insert_documents(&mut wtxn, &index, content); let deleted_external_ids = ["1_7", "1_52"]; let deleted_internal_ids = delete_documents(&mut wtxn, &index, &deleted_external_ids); @@ -1042,18 +1024,17 @@ mod tests { #[test] fn stats_should_not_return_deleted_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key(S("docid")); - builder.execute(|_| ()).unwrap(); - let content = documents!([ + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_primary_key(S("docid")); + }) + .unwrap(); + + index.add_documents_using_wtxn(&mut wtxn, documents!([ { "docid": "1_4", "label": "sign"}, { "docid": "1_5", "label": "letter"}, { "docid": "1_7", "label": "abstract,cartoon,design,pattern", "title": "Mickey Mouse"}, @@ -1074,9 +1055,8 @@ mod tests { { "docid": "1_58", "label": "abstract,art,cartoon"}, { "docid": "1_68", "label": "design"}, { "docid": "1_69", "label": "geometry"} - ]); + ])).unwrap(); - insert_documents(&mut wtxn, &index, content); delete_documents(&mut wtxn, &index, &["1_7", "1_52"]); // count internal documents diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 63aec1290..b0cae600f 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -604,41 +604,27 @@ fn execute_word_prefix_docids( #[cfg(test)] mod tests { - use std::io::Cursor; - use big_s::S; - use heed::EnvOpenOptions; use maplit::hashset; use super::*; - use crate::documents::DocumentsBatchBuilder; + use crate::documents::documents_batch_reader_from_objects; + use crate::index::tests::TempIndex; use crate::update::DeleteDocuments; use crate::BEU16; #[test] fn simple_document_replacement() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); // First we send 3 documents with ids from 1 to 3. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 1, "name": "kevin" }, - { "id": 2, "name": "kevina" }, - { "id": 3, "name": "benoit" } - ]); - - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 1, "name": "kevin" }, + { "id": 2, "name": "kevina" }, + { "id": 3, "name": "benoit" } + ])) + .unwrap(); // Check that there is 3 documents now. let rtxn = index.read_txn().unwrap(); @@ -647,15 +633,7 @@ mod tests { drop(rtxn); // Second we send 1 document with id 1, to erase the previous ones. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ { "id": 1, "name": "updated kevin" } ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index.add_documents(documents!([ { "id": 1, "name": "updated kevin" } ])).unwrap(); // Check that there is **always** 3 documents. let rtxn = index.read_txn().unwrap(); @@ -664,18 +642,13 @@ mod tests { drop(rtxn); // Third we send 3 documents again to replace the existing ones. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 1, "name": "updated second kevin" }, - { "id": 2, "name": "updated kevina" }, - { "id": 3, "name": "updated benoit" } - ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 1, "name": "updated second kevin" }, + { "id": 2, "name": "updated kevina" }, + { "id": 3, "name": "updated benoit" } + ])) + .unwrap(); // Check that there is **always** 3 documents. let rtxn = index.read_txn().unwrap(); @@ -686,31 +659,18 @@ mod tests { #[test] fn simple_document_merge() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; // First we send 3 documents with duplicate ids and // change the index method to merge documents. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 1, "name": "kevin" }, - { "id": 1, "name": "kevina" }, - { "id": 1, "name": "benoit" } - ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig { - update_method: IndexDocumentsMethod::UpdateDocuments, - ..Default::default() - }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 1, "name": "kevin" }, + { "id": 1, "name": "kevina" }, + { "id": 1, "name": "benoit" } + ])) + .unwrap(); // Check that there is only 1 document now. let rtxn = index.read_txn().unwrap(); @@ -731,14 +691,7 @@ mod tests { drop(rtxn); // Second we send 1 document with id 1, to force it to be merged with the previous one. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ { "id": 1, "age": 25 } ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index.add_documents(documents!([ { "id": 1, "age": 25 } ])).unwrap(); // Check that there is **always** 1 document. let rtxn = index.read_txn().unwrap(); @@ -763,25 +716,14 @@ mod tests { #[test] fn not_auto_generated_documents_ids() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); - // First we send 3 documents with ids from 1 to 3. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ + let result = index.add_documents(documents!([ { "name": "kevin" }, { "name": "kevina" }, { "name": "benoit" } - ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (_builder, user_error) = builder.add_documents(content).unwrap(); - assert!(user_error.is_err()); - wtxn.commit().unwrap(); + ])); + assert!(result.is_err()); // Check that there is no document. let rtxn = index.read_txn().unwrap(); @@ -792,28 +734,16 @@ mod tests { #[test] fn simple_auto_generated_documents_ids() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; // First we send 3 documents with ids from 1 to 3. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "name": "kevin" }, - { "name": "kevina" }, - { "name": "benoit" } - ]); - let config = IndexerConfig::default(); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "name": "kevin" }, + { "name": "kevina" }, + { "name": "benoit" } + ])) + .unwrap(); // Check that there is 3 documents now. let rtxn = index.read_txn().unwrap(); @@ -826,14 +756,7 @@ mod tests { drop(rtxn); // Second we send 1 document with the generated uuid, to erase the previous ones. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ { "name": "updated kevin", "id": kevin_uuid } ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index.add_documents(documents!([ { "name": "updated kevin", "id": kevin_uuid } ])).unwrap(); // Check that there is **always** 3 documents. let rtxn = index.read_txn().unwrap(); @@ -857,26 +780,16 @@ mod tests { #[test] fn reordered_auto_generated_documents_ids() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); // First we send 3 documents with ids from 1 to 3. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 1, "name": "kevin" }, - { "id": 2, "name": "kevina" }, - { "id": 3, "name": "benoit" } - ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 1, "name": "kevin" }, + { "id": 2, "name": "kevina" }, + { "id": 3, "name": "benoit" } + ])) + .unwrap(); // Check that there is 3 documents now. let rtxn = index.read_txn().unwrap(); @@ -885,16 +798,8 @@ mod tests { drop(rtxn); // Second we send 1 document without specifying the id. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ { "name": "new kevin" } ]); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index.index_documents_config.autogenerate_docids = true; + index.add_documents(documents!([ { "name": "new kevin" } ])).unwrap(); // Check that there is 4 documents now. let rtxn = index.read_txn().unwrap(); @@ -905,22 +810,10 @@ mod tests { #[test] fn empty_update() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); // First we send 0 documents and only headers. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index.add_documents(documents!([])).unwrap(); // Check that there is no documents. let rtxn = index.read_txn().unwrap(); @@ -931,34 +824,14 @@ mod tests { #[test] fn invalid_documents_ids() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); // First we send 1 document with an invalid id. - let mut wtxn = index.write_txn().unwrap(); // There is a space in the document id. - let content = documents!([ { "id": "brume bleue", "name": "kevin" } ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (_builder, user_error) = builder.add_documents(content).unwrap(); - assert!(user_error.is_err()); - wtxn.commit().unwrap(); + index.add_documents(documents!([ { "id": "brume bleue", "name": "kevin" } ])).unwrap_err(); - // First we send 1 document with a valid id. - let mut wtxn = index.write_txn().unwrap(); - // There is a space in the document id. - let content = documents!([ { "id": 32, "name": "kevin" } ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + // Then we send 1 document with a valid id. + index.add_documents(documents!([ { "id": 32, "name": "kevin" } ])).unwrap(); // Check that there is 1 document now. let rtxn = index.read_txn().unwrap(); @@ -969,26 +842,16 @@ mod tests { #[test] fn complex_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); // First we send 3 documents with an id for only one of them. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } }, - { "id": 1, "name": "kevina", "array": ["I", "am", "fine"] }, - { "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] } - ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } }, + { "id": 1, "name": "kevina", "array": ["I", "am", "fine"] }, + { "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] } + ])) + .unwrap(); // Check that there is 1 documents now. let rtxn = index.read_txn().unwrap(); @@ -1010,126 +873,72 @@ mod tests { #[test] fn simple_documents_replace() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; - // First we send 3 documents with an id for only one of them. - let mut wtxn = index.write_txn().unwrap(); - let documents = documents!([ + index.add_documents(documents!([ { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance", "price": 3.5, "_geo": { "lat": 12, "lng": 42 } }, { "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" , "price": 10.0 }, { "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy", "price": 25.99 }, { "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" }, { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "_geo": { "lat": 35, "lng": 23 } } - ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig { - update_method: IndexDocumentsMethod::ReplaceDocuments, - ..Default::default() - }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + ])).unwrap(); - let mut wtxn = index.write_txn().unwrap(); - let indexing_config = IndexDocumentsConfig { - update_method: IndexDocumentsMethod::UpdateDocuments, - ..Default::default() - }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let documents = documents!([ - { - "id": 2, - "author": "J. Austen", - "date": "1813" - } - ]); + index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([{ + "id": 2, + "author": "J. Austen", + "date": "1813" + }])) + .unwrap(); } #[test] fn mixed_geo_documents() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; // We send 6 documents and mix the ones that have _geo and those that don't have it. - let mut wtxn = index.write_txn().unwrap(); - let documents = documents!([ - { "id": 2, "price": 3.5, "_geo": { "lat": 12, "lng": 42 } }, - { "id": 456 }, - { "id": 1 }, - { "id": 1344 }, - { "id": 4 }, - { "id": 42, "_geo": { "lat": 35, "lng": 23 } } - ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig { - update_method: IndexDocumentsMethod::ReplaceDocuments, - ..Default::default() - }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 2, "price": 3.5, "_geo": { "lat": 12, "lng": 42 } }, + { "id": 456 }, + { "id": 1 }, + { "id": 1344 }, + { "id": 4 }, + { "id": 42, "_geo": { "lat": 35, "lng": 23 } } + ])) + .unwrap(); - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - - let faceted_fields = hashset!(S("_geo")); - builder.set_filterable_fields(faceted_fields); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_filterable_fields(hashset!(S("_geo"))); + }) + .unwrap(); } #[test] fn index_all_flavour_of_geo() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; - let config = IndexerConfig::default(); - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); + index + .update_settings(|settings| { + settings.set_filterable_fields(hashset!(S("_geo"))); + }) + .unwrap(); - builder.set_filterable_fields(hashset!(S("_geo"))); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); - - let indexing_config = IndexDocumentsConfig { - update_method: IndexDocumentsMethod::ReplaceDocuments, - ..Default::default() - }; - let mut wtxn = index.write_txn().unwrap(); - - let documents = documents!([ - { "id": 0, "_geo": { "lat": 31, "lng": [42] } }, - { "id": 1, "_geo": { "lat": "31" }, "_geo.lng": 42 }, - { "id": 2, "_geo": { "lng": "42" }, "_geo.lat": "31" }, - { "id": 3, "_geo.lat": 31, "_geo.lng": "42" }, - ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "id": 0, "_geo": { "lat": 31, "lng": [42] } }, + { "id": 1, "_geo": { "lat": "31" }, "_geo.lng": 42 }, + { "id": 2, "_geo": { "lng": "42" }, "_geo.lat": "31" }, + { "id": 3, "_geo.lat": 31, "_geo.lng": "42" }, + ])) + .unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1141,90 +950,60 @@ mod tests { #[test] fn geo_error() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; - let config = IndexerConfig::default(); - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); + index + .update_settings(|settings| { + settings.set_filterable_fields(hashset!(S("_geo"))); + }) + .unwrap(); - builder.set_filterable_fields(hashset!(S("_geo"))); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); - - let indexing_config = IndexDocumentsConfig { - update_method: IndexDocumentsMethod::ReplaceDocuments, - ..Default::default() - }; - let mut wtxn = index.write_txn().unwrap(); - - let documents = documents!([ - { "id": 0, "_geo": { "lng": 42 } } - ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - let error = builder.execute().unwrap_err(); + let error = index + .add_documents(documents!([ + { "id": 0, "_geo": { "lng": 42 } } + ])) + .unwrap_err(); assert_eq!( &error.to_string(), r#"Could not find latitude in the document with the id: `0`. Was expecting a `_geo.lat` field."# ); - let documents = documents!([ - { "id": 0, "_geo": { "lat": 42 } } - ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - let error = builder.execute().unwrap_err(); + let error = index + .add_documents(documents!([ + { "id": 0, "_geo": { "lat": 42 } } + ])) + .unwrap_err(); assert_eq!( &error.to_string(), r#"Could not find longitude in the document with the id: `0`. Was expecting a `_geo.lng` field."# ); - let documents = documents!([ - { "id": 0, "_geo": { "lat": "lol", "lng": 42 } } - ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - let error = builder.execute().unwrap_err(); + let error = index + .add_documents(documents!([ + { "id": 0, "_geo": { "lat": "lol", "lng": 42 } } + ])) + .unwrap_err(); assert_eq!( &error.to_string(), r#"Could not parse latitude in the document with the id: `0`. Was expecting a finite number but instead got `"lol"`."# ); - let documents = documents!([ - { "id": 0, "_geo": { "lat": [12, 13], "lng": 42 } } - ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - let error = builder.execute().unwrap_err(); + let error = index + .add_documents(documents!([ + { "id": 0, "_geo": { "lat": [12, 13], "lng": 42 } } + ])) + .unwrap_err(); assert_eq!( &error.to_string(), r#"Could not parse latitude in the document with the id: `0`. Was expecting a finite number but instead got `[12,13]`."# ); - let documents = documents!([ - { "id": 0, "_geo": { "lat": 12, "lng": "hello" } } - ]); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(documents).unwrap(); - user_error.unwrap(); - let error = builder.execute().unwrap_err(); + let error = index + .add_documents(documents!([ + { "id": 0, "_geo": { "lat": 12, "lng": "hello" } } + ])) + .unwrap_err(); assert_eq!( &error.to_string(), r#"Could not parse longitude in the document with the id: `0`. Was expecting a finite number but instead got `"hello"`."# @@ -1233,27 +1012,17 @@ mod tests { #[test] fn delete_documents_then_insert() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); + index + .add_documents(documents!([ + { "objectId": 123, "title": "Pride and Prejudice", "comment": "A great book" }, + { "objectId": 456, "title": "Le Petit Prince", "comment": "A french book" }, + { "objectId": 1, "title": "Alice In Wonderland", "comment": "A weird book" }, + { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } + ])) + .unwrap(); let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "objectId": 123, "title": "Pride and Prejudice", "comment": "A great book" }, - { "objectId": 456, "title": "Le Petit Prince", "comment": "A french book" }, - { "objectId": 1, "title": "Alice In Wonderland", "comment": "A weird book" }, - { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } - ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - assert_eq!(index.primary_key(&wtxn).unwrap(), Some("objectId")); // Delete not all of the documents but some of them. @@ -1263,42 +1032,29 @@ mod tests { let external_documents_ids = index.external_documents_ids(&wtxn).unwrap(); assert!(external_documents_ids.get("30").is_none()); + wtxn.commit().unwrap(); - let content = documents!([ - { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } - ]); + index + .add_documents(documents!([ + { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } + ])) + .unwrap(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); + let wtxn = index.write_txn().unwrap(); let external_documents_ids = index.external_documents_ids(&wtxn).unwrap(); assert!(external_documents_ids.get("30").is_some()); - - let content = documents!([ - { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } - ]); - - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + + index + .add_documents(documents!([ + { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } + ])) + .unwrap(); } #[test] fn index_more_than_256_fields() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - - let mut wtxn = index.write_txn().unwrap(); + let index = TempIndex::new(); let mut big_object = serde_json::Map::new(); big_object.insert(S("id"), serde_json::Value::from("wow")); @@ -1307,56 +1063,23 @@ mod tests { big_object.insert(key, serde_json::Value::from("I am a text!")); } - let mut builder = DocumentsBatchBuilder::new(Vec::new()); - builder.append_json_object(&big_object).unwrap(); - let vector = builder.into_inner().unwrap(); - let content = DocumentsBatchReader::from_reader(Cursor::new(vector)).unwrap(); - - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - - wtxn.commit().unwrap(); + let documents = documents_batch_reader_from_objects([big_object]); + index.add_documents(documents).unwrap(); } #[test] fn index_more_than_1000_positions_in_a_field() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(50 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - - let mut wtxn = index.write_txn().unwrap(); - - let mut big_object = serde_json::Map::new(); - big_object.insert(S("id"), serde_json::Value::from("wow")); - let content: String = (0..=u16::MAX) - .into_iter() - .map(|p| p.to_string()) - .reduce(|a, b| a + " " + b.as_ref()) + let index = TempIndex::new_with_map_size(4096 * 100_000); // 400 MB + let mut content = String::with_capacity(382101); + for i in 0..=u16::MAX { + content.push_str(&format!("{i} ")); + } + index + .add_documents(documents!({ + "id": "wow", + "content": content + })) .unwrap(); - big_object.insert("content".to_string(), serde_json::Value::from(content)); - - let mut builder = DocumentsBatchBuilder::new(Vec::new()); - builder.append_json_object(&big_object).unwrap(); - let vector = builder.into_inner().unwrap(); - let content = DocumentsBatchReader::from_reader(Cursor::new(vector)).unwrap(); - - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - - wtxn.commit().unwrap(); let mut rtxn = index.read_txn().unwrap(); @@ -1370,117 +1093,90 @@ mod tests { #[test] fn index_documents_with_zeroes() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { - "id": 2, - "title": "Prideand Prejudice", - "au{hor": "Jane Austin", - "genre": "romance", - "price$": "3.5$", - }, - { - "id": 456, - "title": "Le Petit Prince", - "au{hor": "Antoine de Saint-Exupéry", - "genre": "adventure", - "price$": "10.0$", - }, - { - "id": 1, - "title": "Wonderland", - "au{hor": "Lewis Carroll", - "genre": "fantasy", - "price$": "25.99$", - }, - { - "id": 4, - "title": "Harry Potter ing fantasy\0lood Prince", - "au{hor": "J. K. Rowling", - "genre": "fantasy\0", - }, - ]); - - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { + "id": 2, + "title": "Prideand Prejudice", + "au{hor": "Jane Austin", + "genre": "romance", + "price$": "3.5$", + }, + { + "id": 456, + "title": "Le Petit Prince", + "au{hor": "Antoine de Saint-Exupéry", + "genre": "adventure", + "price$": "10.0$", + }, + { + "id": 1, + "title": "Wonderland", + "au{hor": "Lewis Carroll", + "genre": "fantasy", + "price$": "25.99$", + }, + { + "id": 4, + "title": "Harry Potter ing fantasy\0lood Prince", + "au{hor": "J. K. Rowling", + "genre": "fantasy\0", + }, + ])) + .unwrap(); } #[test] fn index_documents_with_nested_fields() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { - "id": 0, - "title": "The zeroth document", - }, - { - "id": 1, - "title": "The first document", - "nested": { - "object": "field", - "machin": "bidule", + index + .add_documents(documents!([ + { + "id": 0, + "title": "The zeroth document", }, - }, - { - "id": 2, - "title": "The second document", - "nested": [ - "array", - { + { + "id": 1, + "title": "The first document", + "nested": { "object": "field", + "machin": "bidule", }, - { - "prout": "truc", - "machin": "lol", - }, - ], - }, - { - "id": 3, - "title": "The third document", - "nested": "I lied", - }, - ]); + }, + { + "id": 2, + "title": "The second document", + "nested": [ + "array", + { + "object": "field", + }, + { + "prout": "truc", + "machin": "lol", + }, + ], + }, + { + "id": 3, + "title": "The third document", + "nested": "I lied", + }, + ])) + .unwrap(); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); + index + .update_settings(|settings| { + let searchable_fields = vec![S("title"), S("nested.object"), S("nested.machin")]; + settings.set_searchable_fields(searchable_fields); - wtxn.commit().unwrap(); - - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - - let searchable_fields = vec![S("title"), S("nested.object"), S("nested.machin")]; - builder.set_searchable_fields(searchable_fields); - - let faceted_fields = hashset!(S("title"), S("nested.object"), S("nested.machin")); - builder.set_filterable_fields(faceted_fields); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + let faceted_fields = hashset!(S("title"), S("nested.object"), S("nested.machin")); + settings.set_filterable_fields(faceted_fields); + }) + .unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1554,54 +1250,42 @@ mod tests { #[test] fn index_documents_with_nested_primary_key() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let index = TempIndex::new(); - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key("complex.nested.id".to_owned()); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_primary_key("complex.nested.id".to_owned()); + }) + .unwrap(); - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { - "complex": { - "nested": { - "id": 0, + index + .add_documents(documents!([ + { + "complex": { + "nested": { + "id": 0, + }, }, + "title": "The zeroth document", }, - "title": "The zeroth document", - }, - { - "complex.nested": { - "id": 1, + { + "complex.nested": { + "id": 1, + }, + "title": "The first document", }, - "title": "The first document", - }, - { - "complex": { - "nested.id": 2, + { + "complex": { + "nested.id": 2, + }, + "title": "The second document", }, - "title": "The second document", - }, - { - "complex.nested.id": 3, - "title": "The third document", - }, - ]); - - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + { + "complex.nested.id": 3, + "title": "The third document", + }, + ])) + .unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1630,50 +1314,28 @@ mod tests { #[test] fn retrieve_a_b_nested_document_id() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let index = TempIndex::new(); - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key("a.b".to_owned()); - builder.execute(|_| ()).unwrap(); - - let content = documents!({ "a" : { "b" : { "c" : 1 }}}); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (_builder, user_error) = builder.add_documents(content).unwrap(); + index + .update_settings(|settings| { + settings.set_primary_key("a.b".to_owned()); + }) + .unwrap(); // There must be an issue with the primary key no present in the given document - user_error.unwrap_err(); + index.add_documents(documents!({ "a" : { "b" : { "c" : 1 }}})).unwrap_err(); } #[test] fn retrieve_a_b_c_nested_document_id() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let index = TempIndex::new(); - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key("a.b.c".to_owned()); - builder.execute(|_| ()).unwrap(); - - let content = documents!({ "a" : { "b" : { "c" : 1 }}}); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_primary_key("a.b.c".to_owned()); + }) + .unwrap(); + index.add_documents(documents!({ "a" : { "b" : { "c" : 1 }}})).unwrap(); let rtxn = index.read_txn().unwrap(); let external_documents_ids = index.external_documents_ids(&rtxn).unwrap(); @@ -1682,61 +1344,42 @@ mod tests { #[test] fn test_facets_generation() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { - "id": 0, - "dog": { - "race": { - "bernese mountain": "zeroth", + index + .add_documents(documents!([ + { + "id": 0, + "dog": { + "race": { + "bernese mountain": "zeroth", + }, }, }, - }, - { - "id": 1, - "dog.race": { - "bernese mountain": "first", + { + "id": 1, + "dog.race": { + "bernese mountain": "first", + }, }, - }, - { - "id": 2, - "dog.race.bernese mountain": "second", - }, - { - "id": 3, - "dog": { - "race.bernese mountain": "third" + { + "id": 2, + "dog.race.bernese mountain": "second", }, - }, - ]); + { + "id": 3, + "dog": { + "race.bernese mountain": "third" + }, + }, + ])) + .unwrap(); - // index the documents - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - - wtxn.commit().unwrap(); - - // ---- ADD THE SETTING TO TEST THE FILTERABLE - - // add the settings - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - - builder.set_filterable_fields(hashset!(String::from("dog"))); - - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_filterable_fields(hashset!(String::from("dog"))); + }) + .unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1751,17 +1394,12 @@ mod tests { let crate::SearchResult { documents_ids, .. } = search.execute().unwrap(); assert_eq!(documents_ids, vec![i]); } - - // ---- RESET THE SETTINGS - - // update the settings - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - - builder.reset_filterable_fields(); - - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + // Reset the settings + index + .update_settings(|settings| { + settings.reset_filterable_fields(); + }) + .unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1769,16 +1407,12 @@ mod tests { assert_eq!(facets, hashset!()); - // ---- UPDATE THE SETTINGS TO TEST THE SORTABLE - - // update the settings - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - - builder.set_sortable_fields(hashset!(S("dog.race"))); - - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + // update the settings to test the sortable + index + .update_settings(|settings| { + settings.set_sortable_fields(hashset!(S("dog.race"))); + }) + .unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1796,69 +1430,37 @@ mod tests { #[test] fn index_2_times_documents_split_by_zero_document_indexation() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); - let content = documents!([ - {"id": 0, "name": "Kerollmops", "score": 78}, - {"id": 1, "name": "ManyTheFish", "score": 75}, - {"id": 2, "name": "Ferdi", "score": 39}, - {"id": 3, "name": "Tommy", "score": 33} - ]); - - let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + {"id": 0, "name": "Kerollmops", "score": 78}, + {"id": 1, "name": "ManyTheFish", "score": 75}, + {"id": 2, "name": "Ferdi", "score": 39}, + {"id": 3, "name": "Tommy", "score": 33} + ])) + .unwrap(); // Check that there is 4 document now. let rtxn = index.read_txn().unwrap(); let count = index.number_of_documents(&rtxn).unwrap(); assert_eq!(count, 4); - let content = documents!([]); - - let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index.add_documents(documents!([])).unwrap(); // Check that there is 4 document now. let rtxn = index.read_txn().unwrap(); let count = index.number_of_documents(&rtxn).unwrap(); assert_eq!(count, 4); - let content = documents!([ - {"id": 0, "name": "Kerollmops", "score": 78}, - {"id": 1, "name": "ManyTheFish", "score": 75}, - {"id": 2, "name": "Ferdi", "score": 39}, - {"id": 3, "name": "Tommy", "score": 33} - ]); - - let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + {"id": 0, "name": "Kerollmops", "score": 78}, + {"id": 1, "name": "ManyTheFish", "score": 75}, + {"id": 2, "name": "Ferdi", "score": 39}, + {"id": 3, "name": "Tommy", "score": 33} + ])) + .unwrap(); // Check that there is 4 document now. let rtxn = index.read_txn().unwrap(); @@ -1868,26 +1470,14 @@ mod tests { #[test] fn test_meilisearch_1714() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); - let content = documents!([ - {"id": "123", "title": "小化妆包" }, - {"id": "456", "title": "Ipad 包" } - ]); - - let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + {"id": "123", "title": "小化妆包" }, + {"id": "456", "title": "Ipad 包" } + ])) + .unwrap(); let rtxn = index.read_txn().unwrap(); @@ -1913,35 +1503,20 @@ mod tests { /// it should not return any error. #[test] fn text_with_too_long_words() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); - let content = documents!([ - {"id": 1, "title": "a".repeat(256) }, - {"id": 2, "title": "b".repeat(512) }, - {"id": 3, "title": format!("{} {}", "c".repeat(250), "d".repeat(250)) }, - ]); - - let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + {"id": 1, "title": "a".repeat(256) }, + {"id": 2, "title": "b".repeat(512) }, + {"id": 3, "title": format!("{} {}", "c".repeat(250), "d".repeat(250)) }, + ])) + .unwrap(); } #[test] fn text_with_too_long_keys() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); let script = "https://bug.example.com/meilisearch/milli.saml2?ROLE=Programmer-1337&SAMLRequest=Cy1ytcZT1Po%2L2IY2y9Unru8rgnW4qWfPiI0EpT7P8xjJV8PeQikRL%2E8D9A4pj9tmbymbQCQwGmGjPMK7qwXFPX4DH52JO2b7n6TXjuR7zkIFuYdzdY2rwRNBPgCL7ihclEm9zyIjKZQ%2JTqiwfXxWjnI0KEYQYHdwd6Q%2Fx%28BDLNsvmL54CCY2F4RWeRs4eqWfn%2EHqxlhreFzax4AiQ2tgOtV5thOaaWqrhZD%2Py70nuyZWNTKwciGI43AoHg6PThANsQ5rAY5amzN%2ufbs1swETUXlLZuOut5YGpYPZfY6STJWNp4QYSUOUXBZpdElYsH7UHZ7VhJycgyt%28aTK0GW6GbKne2tJM0hgSczOqndg6RFa9WsnSBi4zMcaEfYur4WlSsHDYInF9ROousKqVMZ6H8%2gbUissaLh1eXRGo8KEJbyEHbhVVKGD%28kx4cfKjx9fT3pkeDTdvDrVn25jIzi9wHyt9l1lWc8ICnCvXCVUPP%2BjBG4wILR29gMV9Ux2QOieQm2%2Fycybhr8sBGCl30mHC7blvWt%2T3mrCHQoS3VK49PZNPqBZO9C7vOjOWoszNkJx4QckWV%2FZFvbpzUUkiBiehr9F%2FvQSxz9lzv68GwbTu9fr638p%2FQM%3D&RelayState=https%3A%2F%example.bug.com%2Fde&SigAlg=http%3A%2F%2Fwww.w3.org%2F2000%2F09%2Fxmldsig%23rsa-sha1&Signature=AZFpkhFFII7PodiewTovaGnLQKUVZp0qOCCcBIUkJ6P5by3lE3Lldj9pKaFu4wz4j%2B015HEhDvF0LlAmwwES85vdGh%2FpD%2cIQPRUEjdCbQkQDd3dy1mMXbpXxSe4QYcv9Ni7tqNTQxekpO1gE7rtg6zC66EU55uM9aj9abGQ034Vly%2F6IJ08bvAq%2B%2FB9KruLstuiNWnlXTfNGsOxGLK7%2BXr94LTkat8m%2FMan6Qr95%2KeR5TmmqaQIE4N9H6o4TopT7mXr5CF2Z3"; // Create 200 documents with a long text @@ -1953,60 +1528,22 @@ mod tests { serde_json::Value::Object(object) => Some(object), _ => None, }); - - let mut builder = crate::documents::DocumentsBatchBuilder::new(Vec::new()); - for object in documents_iter { - builder.append_json_object(&object).unwrap(); - } - let vector = builder.into_inner().unwrap(); - crate::documents::DocumentsBatchReader::from_reader(Cursor::new(vector)).unwrap() + documents_batch_reader_from_objects(documents_iter) }; - // Index those 200 long documents - let mut wtxn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); + index.add_documents(content).unwrap(); - // Create one long document - let content = documents!([ - {"id": 400, "script": script }, - ]); - - // Index this one long document - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - - wtxn.commit().unwrap(); + // Index one long document + index + .add_documents(documents!([ + {"id": 400, "script": script }, + ])) + .unwrap(); } #[test] fn index_documents_in_multiple_transforms() { - let tmp = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let index = Index::new(options, tmp).unwrap(); - let mut wtxn = index.write_txn().unwrap(); - let indexer_config = IndexerConfig::default(); - let builder = IndexDocuments::new( - &mut wtxn, - &index, - &indexer_config, - IndexDocumentsConfig::default(), - |_| (), - ) - .unwrap(); + let index = TempIndex::new(); let doc1 = documents! {[{ "id": 228142, @@ -2028,12 +1565,10 @@ mod tests { "branch_id_number": 0 }]}; - let (builder, user_error) = builder.add_documents(doc1).unwrap(); - user_error.unwrap(); - let (builder, user_error) = builder.add_documents(doc2).unwrap(); - user_error.unwrap(); + index.add_documents(doc1).unwrap(); + index.add_documents(doc2).unwrap(); - builder.execute().unwrap(); + let wtxn = index.read_txn().unwrap(); let map = index.external_documents_ids(&wtxn).unwrap().to_hash_map(); let ids = map.values().collect::>(); @@ -2043,10 +1578,7 @@ mod tests { #[test] fn index_documents_check_exists_database() { - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let faceted_fields = hashset!(S("colour")); let content = || { documents!([ { @@ -2088,30 +1620,6 @@ mod tests { } ]) }; - let make_index = || { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - Index::new(options, &path).unwrap() - }; - - let set_filterable_fields = |index: &Index| { - let mut wtxn = index.write_txn().unwrap(); - let mut builder = update::Settings::new(&mut wtxn, &index, &config); - builder.set_filterable_fields(faceted_fields.clone()); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); - }; - let add_documents = |index: &Index| { - let mut wtxn = index.write_txn().unwrap(); - let builder = - IndexDocuments::new(&mut wtxn, index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content()).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); - }; let check_ok = |index: &Index| { let rtxn = index.read_txn().unwrap(); @@ -2132,34 +1640,27 @@ mod tests { .unwrap(); assert_eq!(bitmap_colour_green.into_iter().collect::>(), vec![6, 7]); }; + + let faceted_fields = hashset!(S("colour")); - let index = make_index(); - add_documents(&index); - set_filterable_fields(&index); + let index = TempIndex::new(); + index.add_documents(content()).unwrap(); + index.update_settings(|settings| { + settings.set_filterable_fields(faceted_fields.clone()); + }).unwrap(); check_ok(&index); - let index = make_index(); - set_filterable_fields(&index); - add_documents(&index); + let index = TempIndex::new(); + index.update_settings(|settings| { + settings.set_filterable_fields(faceted_fields.clone()); + }).unwrap(); + index.add_documents(content()).unwrap(); check_ok(&index); } #[test] fn primary_key_must_not_contain_floats() { - let tmp = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let index = Index::new(options, tmp).unwrap(); - let mut wtxn = index.write_txn().unwrap(); - let indexer_config = IndexerConfig::default(); - let builder = IndexDocuments::new( - &mut wtxn, - &index, - &indexer_config, - IndexDocumentsConfig::default(), - |_| (), - ) - .unwrap(); + let index = TempIndex::new_with_map_size(4096 * 100); let doc1 = documents! {[{ "id": -228142, @@ -2181,32 +1682,15 @@ mod tests { "title": "something", }]}; - let (builder, user_error) = builder.add_documents(doc1).unwrap(); - user_error.unwrap(); - let (builder, user_error) = builder.add_documents(doc2).unwrap(); - assert!(user_error.is_err()); - let (builder, user_error) = builder.add_documents(doc3).unwrap(); - assert!(user_error.is_err()); - let (_builder, user_error) = builder.add_documents(doc4).unwrap(); - assert!(user_error.is_err()); + index.add_documents(doc1).unwrap(); + index.add_documents(doc2).unwrap_err(); + index.add_documents(doc3).unwrap_err(); + index.add_documents(doc4).unwrap_err(); } #[test] fn primary_key_must_not_contain_whitespace() { - let tmp = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(4096 * 100); - let index = Index::new(options, tmp).unwrap(); - let mut wtxn = index.write_txn().unwrap(); - let indexer_config = IndexerConfig::default(); - let builder = IndexDocuments::new( - &mut wtxn, - &index, - &indexer_config, - IndexDocumentsConfig::default(), - |_| (), - ) - .unwrap(); + let index = TempIndex::new(); let doc1 = documents! {[{ "id": " 1", @@ -2228,13 +1712,9 @@ mod tests { "title": "something", }]}; - let (builder, user_error) = builder.add_documents(doc1).unwrap(); - assert!(user_error.is_err()); - let (builder, user_error) = builder.add_documents(doc2).unwrap(); - assert!(user_error.is_err()); - let (builder, user_error) = builder.add_documents(doc3).unwrap(); - assert!(user_error.is_err()); - let (_builder, user_error) = builder.add_documents(doc4).unwrap(); - assert!(user_error.is_err()); + index.add_documents(doc1).unwrap_err(); + index.add_documents(doc2).unwrap_err(); + index.add_documents(doc3).unwrap_err(); + index.add_documents(doc4).unwrap_err(); } } diff --git a/milli/src/update/settings.rs b/milli/src/update/settings.rs index 5f39579b7..0f611572e 100644 --- a/milli/src/update/settings.rs +++ b/milli/src/update/settings.rs @@ -709,45 +709,38 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> { mod tests { use big_s::S; use heed::types::ByteSlice; - use heed::EnvOpenOptions; use maplit::{btreeset, hashmap, hashset}; use super::*; use crate::error::Error; use crate::index::tests::TempIndex; - use crate::update::IndexDocuments; use crate::{Criterion, Filter, SearchResult}; #[test] fn set_and_reset_searchable_fields() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = TempIndex::new(); // First we send 3 documents with ids from 1 to 3. let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "id": 1, "name": "kevin", "age": 23 }, - { "id": 2, "name": "kevina", "age": 21}, - { "id": 3, "name": "benoit", "age": 34 } - ]); - let config = IndexerConfig::default(); - let indexing_config = IndexDocumentsConfig::default(); - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "id": 1, "name": "kevin", "age": 23 }, + { "id": 2, "name": "kevina", "age": 21}, + { "id": 3, "name": "benoit", "age": 34 } + ]), + ) + .unwrap(); // We change the searchable fields to be the "name" field only. - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_searchable_fields(vec!["name".into()]); - builder.execute(|_| ()).unwrap(); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_searchable_fields(vec!["name".into()]); + }) + .unwrap(); + wtxn.commit().unwrap(); // Check that the searchable field is correctly set to "name" only. @@ -766,11 +759,11 @@ mod tests { drop(rtxn); // We change the searchable fields to be the "name" field only. - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.reset_searchable_fields(); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.reset_searchable_fields(); + }) + .unwrap(); // Check that the searchable field have been reset and documents are found now. let rtxn = index.read_txn().unwrap(); @@ -784,36 +777,30 @@ mod tests { #[test] fn mixup_searchable_with_displayed_fields() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; - // First we send 3 documents with ids from 1 to 3. let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "name": "kevin", "age": 23}, - { "name": "kevina", "age": 21 }, - { "name": "benoit", "age": 34 } - ]); - let config = IndexerConfig::default(); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + // First we send 3 documents with ids from 1 to 3. + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "name": "kevin", "age": 23}, + { "name": "kevina", "age": 21 }, + { "name": "benoit", "age": 34 } + ]), + ) + .unwrap(); // In the same transaction we change the displayed fields to be only the "age". // We also change the searchable fields to be the "name" field only. - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_displayed_fields(vec!["age".into()]); - builder.set_searchable_fields(vec!["name".into()]); - builder.execute(|_| ()).unwrap(); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_displayed_fields(vec!["age".into()]); + settings.set_searchable_fields(vec!["name".into()]); + }) + .unwrap(); wtxn.commit().unwrap(); // Check that the displayed fields are correctly set to `None` (default value). @@ -823,11 +810,11 @@ mod tests { drop(rtxn); // We change the searchable fields to be the "name" field only. - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.reset_searchable_fields(); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.reset_searchable_fields(); + }) + .unwrap(); // Check that the displayed fields always contains only the "age" field. let rtxn = index.read_txn().unwrap(); @@ -837,28 +824,17 @@ mod tests { #[test] fn default_displayed_fields() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; // First we send 3 documents with ids from 1 to 3. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "name": "kevin", "age": 23}, - { "name": "kevina", "age": 21 }, - { "name": "benoit", "age": 34 } - ]); - let config = IndexerConfig::default(); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "name": "kevin", "age": 23}, + { "name": "kevina", "age": 21 }, + { "name": "benoit", "age": 34 } + ])) + .unwrap(); // Check that the displayed fields are correctly set to `None` (default value). let rtxn = index.read_txn().unwrap(); @@ -868,32 +844,25 @@ mod tests { #[test] fn set_and_reset_displayed_field() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; - // First we send 3 documents with ids from 1 to 3. let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "name": "kevin", "age": 23}, - { "name": "kevina", "age": 21 }, - { "name": "benoit", "age": 34 } - ]); - let config = IndexerConfig::default(); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - - // In the same transaction we change the displayed fields to be only the age. - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_displayed_fields(vec!["age".into()]); - builder.execute(|_| ()).unwrap(); + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "name": "kevin", "age": 23}, + { "name": "kevina", "age": 21 }, + { "name": "benoit", "age": 34 } + ]), + ) + .unwrap(); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_displayed_fields(vec!["age".into()]); + }) + .unwrap(); wtxn.commit().unwrap(); // Check that the displayed fields are correctly set to only the "age" field. @@ -903,11 +872,11 @@ mod tests { drop(rtxn); // We reset the fields ids to become `None`, the default value. - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.reset_displayed_fields(); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.reset_displayed_fields(); + }) + .unwrap(); // Check that the displayed fields are correctly set to `None` (default value). let rtxn = index.read_txn().unwrap(); @@ -917,34 +886,24 @@ mod tests { #[test] fn set_filterable_fields() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - - let config = IndexerConfig::default(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; // Set the filterable fields to be the age. - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_filterable_fields(hashset! { S("age") }); - builder.execute(|_| ()).unwrap(); + index + .update_settings(|settings| { + settings.set_filterable_fields(hashset! { S("age") }); + }) + .unwrap(); // Then index some documents. - let content = documents!([ - { "name": "kevin", "age": 23}, - { "name": "kevina", "age": 21 }, - { "name": "benoit", "age": 34 } - ]); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "name": "kevin", "age": 23}, + { "name": "kevina", "age": 21 }, + { "name": "benoit", "age": 34 } + ])) + .unwrap(); // Check that the displayed fields are correctly set. let rtxn = index.read_txn().unwrap(); @@ -970,22 +929,13 @@ mod tests { drop(rtxn); // Index a little more documents with new and current facets values. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "name": "kevin2", "age": 23}, - { "name": "kevina2", "age": 21 }, - { "name": "benoit", "age": 35 } - ]); - - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "name": "kevin2", "age": 23}, + { "name": "kevina2", "age": 21 }, + { "name": "benoit", "age": 35 } + ])) + .unwrap(); let rtxn = index.read_txn().unwrap(); // Only count the field_id 0 and level 0 facet values. @@ -1000,35 +950,25 @@ mod tests { #[test] fn set_asc_desc_field() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; // Set the filterable fields to be the age. - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - // Don't display the generated `id` field. - builder.set_displayed_fields(vec![S("name")]); - builder.set_criteria(vec![S("age:asc")]); - builder.execute(|_| ()).unwrap(); + index + .update_settings(|settings| { + settings.set_displayed_fields(vec![S("name")]); + settings.set_criteria(vec![S("age:asc")]); + }) + .unwrap(); // Then index some documents. - let content = documents!([ - { "name": "kevin", "age": 23}, - { "name": "kevina", "age": 21 }, - { "name": "benoit", "age": 34 } - ]); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "name": "kevin", "age": 23}, + { "name": "kevina", "age": 21 }, + { "name": "benoit", "age": 34 } + ])) + .unwrap(); // Run an empty query just to ensure that the search results are ordered. let rtxn = index.read_txn().unwrap(); @@ -1048,39 +988,30 @@ mod tests { #[test] fn set_distinct_field() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; // Set the filterable fields to be the age. - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - // Don't display the generated `id` field. - builder.set_displayed_fields(vec![S("name"), S("age")]); - builder.set_distinct_field(S("age")); - builder.execute(|_| ()).unwrap(); + index + .update_settings(|settings| { + // Don't display the generated `id` field. + settings.set_displayed_fields(vec![S("name"), S("age")]); + settings.set_distinct_field(S("age")); + }) + .unwrap(); // Then index some documents. - let content = documents!([ - { "name": "kevin", "age": 23 }, - { "name": "kevina", "age": 21 }, - { "name": "benoit", "age": 34 }, - { "name": "bernard", "age": 34 }, - { "name": "bertrand", "age": 34 }, - { "name": "bernie", "age": 34 }, - { "name": "ben", "age": 34 } - ]); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "name": "kevin", "age": 23 }, + { "name": "kevina", "age": 21 }, + { "name": "benoit", "age": 34 }, + { "name": "bernard", "age": 34 }, + { "name": "bertrand", "age": 34 }, + { "name": "bernie", "age": 34 }, + { "name": "ben", "age": 34 } + ])) + .unwrap(); // Run an empty query just to ensure that the search results are ordered. let rtxn = index.read_txn().unwrap(); @@ -1092,39 +1023,30 @@ mod tests { #[test] fn set_nested_distinct_field() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; // Set the filterable fields to be the age. - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - // Don't display the generated `id` field. - builder.set_displayed_fields(vec![S("person")]); - builder.set_distinct_field(S("person.age")); - builder.execute(|_| ()).unwrap(); + index + .update_settings(|settings| { + // Don't display the generated `id` field. + settings.set_displayed_fields(vec![S("person")]); + settings.set_distinct_field(S("person.age")); + }) + .unwrap(); // Then index some documents. - let content = documents!([ - { "person": { "name": "kevin", "age": 23 }}, - { "person": { "name": "kevina", "age": 21 }}, - { "person": { "name": "benoit", "age": 34 }}, - { "person": { "name": "bernard", "age": 34 }}, - { "person": { "name": "bertrand", "age": 34 }}, - { "person": { "name": "bernie", "age": 34 }}, - { "person": { "name": "ben", "age": 34 }} - ]); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "person": { "name": "kevin", "age": 23 }}, + { "person": { "name": "kevina", "age": 21 }}, + { "person": { "name": "benoit", "age": 34 }}, + { "person": { "name": "bernard", "age": 34 }}, + { "person": { "name": "bertrand", "age": 34 }}, + { "person": { "name": "bernie", "age": 34 }}, + { "person": { "name": "ben", "age": 34 }} + ])) + .unwrap(); // Run an empty query just to ensure that the search results are ordered. let rtxn = index.read_txn().unwrap(); @@ -1136,28 +1058,17 @@ mod tests { #[test] fn default_stop_words() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; // First we send 3 documents with ids from 1 to 3. - let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "name": "kevin", "age": 23}, - { "name": "kevina", "age": 21 }, - { "name": "benoit", "age": 34 } - ]); - let config = IndexerConfig::default(); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + index + .add_documents(documents!([ + { "name": "kevin", "age": 23}, + { "name": "kevina", "age": 21 }, + { "name": "benoit", "age": 34 } + ])) + .unwrap(); // Ensure there is no stop_words by default let rtxn = index.read_txn().unwrap(); @@ -1167,33 +1078,30 @@ mod tests { #[test] fn set_and_reset_stop_words() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; - // First we send 3 documents with ids from 1 to 3. let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "name": "kevin", "age": 23, "maxim": "I love dogs" }, - { "name": "kevina", "age": 21, "maxim": "Doggos are the best" }, - { "name": "benoit", "age": 34, "maxim": "The crepes are really good" }, - ]); - let config = IndexerConfig::default(); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); + // First we send 3 documents with ids from 1 to 3. + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "name": "kevin", "age": 23, "maxim": "I love dogs" }, + { "name": "kevina", "age": 21, "maxim": "Doggos are the best" }, + { "name": "benoit", "age": 34, "maxim": "The crepes are really good" }, + ]), + ) + .unwrap(); // In the same transaction we provide some stop_words - let mut builder = Settings::new(&mut wtxn, &index, &config); let set = btreeset! { "i".to_string(), "the".to_string(), "are".to_string() }; - builder.set_stop_words(set.clone()); - builder.execute(|_| ()).unwrap(); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_stop_words(set.clone()); + }) + .unwrap(); + wtxn.commit().unwrap(); // Ensure stop_words are effectively stored @@ -1220,11 +1128,11 @@ mod tests { assert_eq!(result.documents_ids.len(), 1); // there is one benoit in our data // now we'll reset the stop_words and ensure it's None - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.reset_stop_words(); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.reset_stop_words(); + }) + .unwrap(); let rtxn = index.read_txn().unwrap(); let stop_words = index.stop_words(&rtxn).unwrap(); @@ -1247,36 +1155,32 @@ mod tests { #[test] fn set_and_reset_synonyms() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; - // Send 3 documents with ids from 1 to 3. let mut wtxn = index.write_txn().unwrap(); - let content = documents!([ - { "name": "kevin", "age": 23, "maxim": "I love dogs"}, - { "name": "kevina", "age": 21, "maxim": "Doggos are the best"}, - { "name": "benoit", "age": 34, "maxim": "The crepes are really good"}, - ]); - let config = IndexerConfig::default(); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); + // Send 3 documents with ids from 1 to 3. + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "name": "kevin", "age": 23, "maxim": "I love dogs"}, + { "name": "kevina", "age": 21, "maxim": "Doggos are the best"}, + { "name": "benoit", "age": 34, "maxim": "The crepes are really good"}, + ]), + ) + .unwrap(); // In the same transaction provide some synonyms - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_synonyms(hashmap! { - "blini".to_string() => vec!["crepes".to_string()], - "super like".to_string() => vec!["love".to_string()], - "puppies".to_string() => vec!["dogs".to_string(), "doggos".to_string()] - }); - builder.execute(|_| ()).unwrap(); + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_synonyms(hashmap! { + "blini".to_string() => vec!["crepes".to_string()], + "super like".to_string() => vec!["love".to_string()], + "puppies".to_string() => vec!["dogs".to_string(), "doggos".to_string()] + }); + }) + .unwrap(); wtxn.commit().unwrap(); // Ensure synonyms are effectively stored @@ -1293,11 +1197,11 @@ mod tests { assert_eq!(result.documents_ids.len(), 2); // Reset the synonyms - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.reset_synonyms(); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.reset_synonyms(); + }) + .unwrap(); // Ensure synonyms are reset let rtxn = index.read_txn().unwrap(); @@ -1315,20 +1219,16 @@ mod tests { #[test] fn setting_searchable_recomputes_other_settings() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let index = TempIndex::new(); // Set all the settings except searchable - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_displayed_fields(vec!["hello".to_string()]); - builder.set_filterable_fields(hashset! { S("age"), S("toto") }); - builder.set_criteria(vec!["toto:asc".to_string()]); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_displayed_fields(vec!["hello".to_string()]); + settings.set_filterable_fields(hashset! { S("age"), S("toto") }); + settings.set_criteria(vec!["toto:asc".to_string()]); + }) + .unwrap(); // check the output let rtxn = index.read_txn().unwrap(); @@ -1339,11 +1239,11 @@ mod tests { drop(rtxn); // We set toto and age as searchable to force reordering of the fields - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_searchable_fields(vec!["toto".to_string(), "age".to_string()]); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_searchable_fields(vec!["toto".to_string(), "age".to_string()]); + }) + .unwrap(); let rtxn = index.read_txn().unwrap(); assert_eq!(&["hello"][..], index.displayed_fields(&rtxn).unwrap().unwrap()); @@ -1353,20 +1253,16 @@ mod tests { #[test] fn setting_not_filterable_cant_filter() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let index = TempIndex::new(); // Set all the settings except searchable - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_displayed_fields(vec!["hello".to_string()]); - // It is only Asc(toto), there is a facet database but it is denied to filter with toto. - builder.set_criteria(vec!["toto:asc".to_string()]); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_displayed_fields(vec!["hello".to_string()]); + // It is only Asc(toto), there is a facet database but it is denied to filter with toto. + settings.set_criteria(vec!["toto:asc".to_string()]); + }) + .unwrap(); let rtxn = index.read_txn().unwrap(); let filter = Filter::from_str("toto = 32").unwrap().unwrap(); @@ -1375,76 +1271,71 @@ mod tests { #[test] fn setting_primary_key() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; - // Set the primary key settings let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key(S("mykey")); - - builder.execute(|_| ()).unwrap(); + // Set the primary key settings + index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.set_primary_key(S("mykey")); + }) + .unwrap(); assert_eq!(index.primary_key(&wtxn).unwrap(), Some("mykey")); // Then index some documents with the "mykey" primary key. - let content = documents!([ - { "mykey": 1, "name": "kevin", "age": 23 }, - { "mykey": 2, "name": "kevina", "age": 21 }, - { "mykey": 3, "name": "benoit", "age": 34 }, - { "mykey": 4, "name": "bernard", "age": 34 }, - { "mykey": 5, "name": "bertrand", "age": 34 }, - { "mykey": 6, "name": "bernie", "age": 34 }, - { "mykey": 7, "name": "ben", "age": 34 } - ]); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); + index + .add_documents_using_wtxn( + &mut wtxn, + documents!([ + { "mykey": 1, "name": "kevin", "age": 23 }, + { "mykey": 2, "name": "kevina", "age": 21 }, + { "mykey": 3, "name": "benoit", "age": 34 }, + { "mykey": 4, "name": "bernard", "age": 34 }, + { "mykey": 5, "name": "bertrand", "age": 34 }, + { "mykey": 6, "name": "bernie", "age": 34 }, + { "mykey": 7, "name": "ben", "age": 34 } + ]), + ) + .unwrap(); wtxn.commit().unwrap(); - // We now try to reset the primary key let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.reset_primary_key(); - - let err = builder.execute(|_| ()).unwrap_err(); - assert!(matches!(err, Error::UserError(UserError::PrimaryKeyCannotBeChanged(_)))); + let error = index + .update_settings_using_wtxn(&mut wtxn, |settings| { + settings.reset_primary_key(); + }) + .unwrap_err(); + assert!(matches!(error, Error::UserError(UserError::PrimaryKeyCannotBeChanged(_)))); wtxn.abort().unwrap(); // But if we clear the database... let mut wtxn = index.write_txn().unwrap(); let builder = ClearDocuments::new(&mut wtxn, &index); builder.execute().unwrap(); + wtxn.commit().unwrap(); // ...we can change the primary key - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_primary_key(S("myid")); - builder.execute(|_| ()).unwrap(); - wtxn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_primary_key(S("myid")); + }) + .unwrap(); } #[test] fn setting_impact_relevancy() { - let path = tempfile::tempdir().unwrap(); - let mut options = EnvOpenOptions::new(); - options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); - let config = IndexerConfig::default(); + let mut index = TempIndex::new(); + index.index_documents_config.autogenerate_docids = true; // Set the genres setting - let mut wtxn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut wtxn, &index, &config); - builder.set_filterable_fields(hashset! { S("genres") }); - builder.execute(|_| ()).unwrap(); + index + .update_settings(|settings| { + settings.set_filterable_fields(hashset! { S("genres") }); + }) + .unwrap(); - let content = documents!([ + index.add_documents(documents!([ { "id": 11, "title": "Star Wars", @@ -1462,18 +1353,8 @@ mod tests { "poster": "https://image.tmdb.org/t/p/w500/gSuHDeWemA1menrwfMRChnSmMVN.jpg", "release_date": 819676800 } - ]); - let indexing_config = - IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; - let builder = - IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) - .unwrap(); - let (builder, user_error) = builder.add_documents(content).unwrap(); - user_error.unwrap(); - builder.execute().unwrap(); - wtxn.commit().unwrap(); + ])).unwrap(); - // We now try to reset the primary key let rtxn = index.read_txn().unwrap(); let SearchResult { documents_ids, .. } = index.search(&rtxn).query("S").execute().unwrap(); let first_id = documents_ids[0]; @@ -1490,45 +1371,41 @@ mod tests { let index = TempIndex::new(); let mut txn = index.write_txn().unwrap(); - let config = IndexerConfig::default(); - assert!(index.authorize_typos(&txn).unwrap()); - let mut builder = Settings::new(&mut txn, &index, &config); - builder.set_autorize_typos(false); - builder.execute(|_| ()).unwrap(); + + index + .update_settings_using_wtxn(&mut txn, |settings| { + settings.set_autorize_typos(false); + }) + .unwrap(); + assert!(!index.authorize_typos(&txn).unwrap()); } #[test] fn update_min_word_len_for_typo() { let index = TempIndex::new(); - let config = IndexerConfig::default(); // Set the genres setting - let mut txn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut txn, &index, &config); - builder.set_min_word_len_one_typo(8); - builder.set_min_word_len_two_typos(8); - builder.execute(|_| ()).unwrap(); - - txn.commit().unwrap(); + index + .update_settings(|settings| { + settings.set_min_word_len_one_typo(8); + settings.set_min_word_len_two_typos(8); + }) + .unwrap(); let txn = index.read_txn().unwrap(); - assert_eq!(index.min_word_len_one_typo(&txn).unwrap(), 8); assert_eq!(index.min_word_len_two_typos(&txn).unwrap(), 8); - let mut txn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut txn, &index, &config); - - builder.reset_min_word_len_one_typo(); - builder.reset_min_word_len_two_typos(); - builder.execute(|_| ()).unwrap(); - - txn.commit().unwrap(); + index + .update_settings(|settings| { + settings.reset_min_word_len_one_typo(); + settings.reset_min_word_len_two_typos(); + }) + .unwrap(); let txn = index.read_txn().unwrap(); - assert_eq!(index.min_word_len_one_typo(&txn).unwrap(), DEFAULT_MIN_WORD_LEN_ONE_TYPO); assert_eq!(index.min_word_len_two_typos(&txn).unwrap(), DEFAULT_MIN_WORD_LEN_TWO_TYPOS); } @@ -1536,28 +1413,29 @@ mod tests { #[test] fn update_invalid_min_word_len_for_typo() { let index = TempIndex::new(); - let config = IndexerConfig::default(); // Set the genres setting - let mut txn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut txn, &index, &config); - builder.set_min_word_len_one_typo(10); - builder.set_min_word_len_two_typos(7); - assert!(builder.execute(|_| ()).is_err()); + index + .update_settings(|settings| { + settings.set_min_word_len_one_typo(10); + settings.set_min_word_len_two_typos(7); + }) + .unwrap_err(); } #[test] fn update_exact_words_normalization() { let index = TempIndex::new(); - let config = IndexerConfig::default(); - // Set the genres setting let mut txn = index.write_txn().unwrap(); - let mut builder = Settings::new(&mut txn, &index, &config); + // Set the genres setting + index + .update_settings_using_wtxn(&mut txn, |settings| { + let words = btreeset! { S("Ab"), S("ac") }; + settings.set_exact_words(words); + }) + .unwrap(); - let words = btreeset! { S("Ab"), S("ac") }; - builder.set_exact_words(words); - assert!(builder.execute(|_| ()).is_ok()); let exact_words = index.exact_words(&txn).unwrap().unwrap(); for word in exact_words.into_fst().stream().into_str_vec().unwrap() { assert!(word.0 == "ac" || word.0 == "ab"); @@ -1567,47 +1445,48 @@ mod tests { #[test] fn test_correct_settings_init() { let index = TempIndex::new(); - let config = IndexerConfig::default(); - let mut txn = index.write_txn().unwrap(); - let builder = Settings::new(&mut txn, &index, &config); - let Settings { - wtxn: _, - index: _, - indexer_config: _, - searchable_fields, - displayed_fields, - filterable_fields, - sortable_fields, - criteria, - stop_words, - distinct_field, - synonyms, - primary_key, - authorize_typos, - min_word_len_two_typos, - min_word_len_one_typo, - exact_words, - exact_attributes, - max_values_per_facet, - pagination_max_total_hits, - } = builder; - - assert!(matches!(searchable_fields, Setting::NotSet)); - assert!(matches!(displayed_fields, Setting::NotSet)); - assert!(matches!(filterable_fields, Setting::NotSet)); - assert!(matches!(sortable_fields, Setting::NotSet)); - assert!(matches!(criteria, Setting::NotSet)); - assert!(matches!(stop_words, Setting::NotSet)); - assert!(matches!(distinct_field, Setting::NotSet)); - assert!(matches!(synonyms, Setting::NotSet)); - assert!(matches!(primary_key, Setting::NotSet)); - assert!(matches!(authorize_typos, Setting::NotSet)); - assert!(matches!(min_word_len_two_typos, Setting::NotSet)); - assert!(matches!(min_word_len_one_typo, Setting::NotSet)); - assert!(matches!(exact_words, Setting::NotSet)); - assert!(matches!(exact_attributes, Setting::NotSet)); - assert!(matches!(max_values_per_facet, Setting::NotSet)); - assert!(matches!(pagination_max_total_hits, Setting::NotSet)); + index + .update_settings(|settings| { + // we don't actually update the settings, just check their content + let Settings { + wtxn: _, + index: _, + indexer_config: _, + searchable_fields, + displayed_fields, + filterable_fields, + sortable_fields, + criteria, + stop_words, + distinct_field, + synonyms, + primary_key, + authorize_typos, + min_word_len_two_typos, + min_word_len_one_typo, + exact_words, + exact_attributes, + max_values_per_facet, + pagination_max_total_hits, + } = settings; + assert!(matches!(searchable_fields, Setting::NotSet)); + assert!(matches!(displayed_fields, Setting::NotSet)); + assert!(matches!(filterable_fields, Setting::NotSet)); + assert!(matches!(sortable_fields, Setting::NotSet)); + assert!(matches!(criteria, Setting::NotSet)); + assert!(matches!(stop_words, Setting::NotSet)); + assert!(matches!(distinct_field, Setting::NotSet)); + assert!(matches!(synonyms, Setting::NotSet)); + assert!(matches!(primary_key, Setting::NotSet)); + assert!(matches!(authorize_typos, Setting::NotSet)); + assert!(matches!(min_word_len_two_typos, Setting::NotSet)); + assert!(matches!(min_word_len_one_typo, Setting::NotSet)); + assert!(matches!(exact_words, Setting::NotSet)); + assert!(matches!(exact_attributes, Setting::NotSet)); + assert!(matches!(max_values_per_facet, Setting::NotSet)); + assert!(matches!(pagination_max_total_hits, Setting::NotSet)); + }) + .unwrap(); } }