snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-30 09:04:59 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Simplify indexing tests

Browse Source
This commit is contained in:
Loïc Lecrenier 2022-08-02 15:13:06 +02:00
parent 21284cf235
commit acff17fb88
6 changed files with 1070 additions and 1726 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
milli/src/documents/mod.rs
View File

@ -179,6 +179,17 @@ macro_rules! documents {
}}; }};
} }
#[cfg(test)]
pub fn documents_batch_reader_from_objects(
objects: impl IntoIterator<Item = Object>,
) -> DocumentsBatchReader<std::io::Cursor<Vec<u8>>> {
let mut builder = DocumentsBatchBuilder::new(Vec::new());
for object in objects {
builder.append_json_object(&object).unwrap();
}
DocumentsBatchReader::from_reader(std::io::Cursor::new(builder.into_inner().unwrap())).unwrap()
}
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use std::io::Cursor; use std::io::Cursor;

229
milli/src/index.rs
View File

@ -1182,16 +1182,19 @@ pub(crate) mod tests {
use std::ops::Deref; use std::ops::Deref;
use big_s::S; use big_s::S;
use heed::EnvOpenOptions; use heed::{EnvOpenOptions, RwTxn};
use maplit::btreemap; use maplit::btreemap;
use tempfile::TempDir; use tempfile::TempDir;
use crate::documents::DocumentsBatchReader;
use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS}; use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS};
use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig}; use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
use crate::Index; use crate::Index;
pub(crate) struct TempIndex { pub(crate) struct TempIndex {
inner: Index, pub inner: Index,
pub indexer_config: IndexerConfig,
pub index_documents_config: IndexDocumentsConfig,
_tempdir: TempDir, _tempdir: TempDir,
} }
@ -1204,43 +1207,88 @@ pub(crate) mod tests {
} }
impl TempIndex { impl TempIndex {
/// Creates a temporary index, with a default `4096 * 100` size. This should be enough for /// Creates a temporary index
/// most tests. pub fn new_with_map_size(size: usize) -> Self {
pub fn new() -> Self {
let mut options = EnvOpenOptions::new(); let mut options = EnvOpenOptions::new();
options.map_size(100 * 4096); options.map_size(size);
let _tempdir = TempDir::new_in(".").unwrap(); let _tempdir = TempDir::new_in(".").unwrap();
let inner = Index::new(options, _tempdir.path()).unwrap(); let inner = Index::new(options, _tempdir.path()).unwrap();
Self { inner, _tempdir } let indexer_config = IndexerConfig::default();
let index_documents_config = IndexDocumentsConfig::default();
Self { inner, indexer_config, index_documents_config, _tempdir }
}
/// Creates a temporary index, with a default `4096 * 1000` size. This should be enough for
/// most tests.
pub fn new() -> Self {
Self::new_with_map_size(4096 * 1000)
}
pub fn add_documents_using_wtxn<'t, R>(
&'t self,
wtxn: &mut RwTxn<'t, '_>,
documents: DocumentsBatchReader<R>,
) -> Result<(), crate::error::Error>
where
R: std::io::Read + std::io::Seek,
{
let builder = IndexDocuments::new(
wtxn,
&self,
&self.indexer_config,
self.index_documents_config.clone(),
|_| (),
)
.unwrap();
let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error?;
builder.execute()?;
Ok(())
}
pub fn add_documents<R>(
&self,
documents: DocumentsBatchReader<R>,
) -> Result<(), crate::error::Error>
where
R: std::io::Read + std::io::Seek,
{
let mut wtxn = self.write_txn().unwrap();
self.add_documents_using_wtxn(&mut wtxn, documents)?;
wtxn.commit().unwrap();
Ok(())
}
pub fn update_settings(
&self,
update: impl Fn(&mut Settings),
) -> Result<(), crate::error::Error> {
let mut wtxn = self.write_txn().unwrap();
self.update_settings_using_wtxn(&mut wtxn, update)?;
wtxn.commit().unwrap();
Ok(())
}
pub fn update_settings_using_wtxn<'t>(
&'t self,
wtxn: &mut RwTxn<'t, '_>,
update: impl Fn(&mut Settings),
) -> Result<(), crate::error::Error> {
let mut builder = update::Settings::new(wtxn, &self.inner, &self.indexer_config);
update(&mut builder);
builder.execute(drop)?;
Ok(())
} }
} }
#[test] #[test]
fn initial_field_distribution() { fn initial_field_distribution() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new(); index
options.map_size(10 * 1024 * 1024); // 10 MB .add_documents(documents!([
let index = Index::new(options, &path).unwrap(); { "id": 1, "name": "kevin" },
{ "id": 2, "name": "bob", "age": 20 },
let mut wtxn = index.write_txn().unwrap(); { "id": 2, "name": "bob", "age": 20 },
let content = documents!([ ]))
{ "id": 1, "name": "kevin" }, .unwrap();
{ "id": 2, "name": "bob", "age": 20 },
{ "id": 2, "name": "bob", "age": 20 },
]);
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
let field_distribution = index.field_distribution(&rtxn).unwrap(); let field_distribution = index.field_distribution(&rtxn).unwrap();
assert_eq!( assert_eq!(
field_distribution, field_distribution,
@ -1253,19 +1301,13 @@ pub(crate) mod tests {
// we add all the documents a second time. we are supposed to get the same // we add all the documents a second time. we are supposed to get the same
// field_distribution in the end // field_distribution in the end
let mut wtxn = index.write_txn().unwrap(); index
let builder = .add_documents(documents!([
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) { "id": 1, "name": "kevin" },
.unwrap(); { "id": 2, "name": "bob", "age": 20 },
let content = documents!([ { "id": 2, "name": "bob", "age": 20 },
{ "id": 1, "name": "kevin" }, ]))
{ "id": 2, "name": "bob", "age": 20 }, .unwrap();
{ "id": 2, "name": "bob", "age": 20 },
]);
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -1280,19 +1322,12 @@ pub(crate) mod tests {
); );
// then we update a document by removing one field and another by adding one field // then we update a document by removing one field and another by adding one field
let content = documents!([ index
{ "id": 1, "name": "kevin", "has_dog": true }, .add_documents(documents!([
{ "id": 2, "name": "bob" } { "id": 1, "name": "kevin", "has_dog": true },
]); { "id": 2, "name": "bob" }
]))
let mut wtxn = index.write_txn().unwrap(); .unwrap();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -1341,35 +1376,19 @@ pub(crate) mod tests {
#[test] #[test]
fn add_documents_and_set_searchable_fields() { fn add_documents_and_set_searchable_fields() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new(); index
options.map_size(10 * 1024 * 1024); // 10 MB .add_documents(documents!([
let index = Index::new(options, &path).unwrap(); { "id": 1, "doggo": "kevin" },
{ "id": 2, "doggo": { "name": "bob", "age": 20 } },
let mut wtxn = index.write_txn().unwrap(); { "id": 3, "name": "jean", "age": 25 },
let content = documents!([ ]))
{ "id": 1, "doggo": "kevin" }, .unwrap();
{ "id": 2, "doggo": { "name": "bob", "age": 20 } }, index
{ "id": 3, "name": "jean", "age": 25 }, .update_settings(|settings| {
]); settings.set_searchable_fields(vec![S("doggo"), S("name")]);
})
let config = IndexerConfig::default(); .unwrap();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// set searchable fields
let mut wtxn = index.write_txn().unwrap();
let mut builder = update::Settings::new(&mut wtxn, &index, &config);
builder.set_searchable_fields(vec![S("doggo"), S("name")]);
builder.execute(drop).unwrap();
wtxn.commit().unwrap();
// ensure we get the right real searchable fields + user defined searchable fields // ensure we get the right real searchable fields + user defined searchable fields
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -1383,19 +1402,13 @@ pub(crate) mod tests {
#[test] #[test]
fn set_searchable_fields_and_add_documents() { fn set_searchable_fields_and_add_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
// set searchable fields index
let mut wtxn = index.write_txn().unwrap(); .update_settings(|settings| {
let mut builder = update::Settings::new(&mut wtxn, &index, &config); settings.set_searchable_fields(vec![S("doggo"), S("name")]);
builder.set_searchable_fields(vec![S("doggo"), S("name")]); })
.unwrap();
builder.execute(drop).unwrap();
wtxn.commit().unwrap();
// ensure we get the right real searchable fields + user defined searchable fields // ensure we get the right real searchable fields + user defined searchable fields
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -1405,21 +1418,13 @@ pub(crate) mod tests {
let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap(); let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap();
assert_eq!(user_defined, &["doggo", "name"]); assert_eq!(user_defined, &["doggo", "name"]);
let mut wtxn = index.write_txn().unwrap(); index
let content = documents!([ .add_documents(documents!([
{ "id": 1, "doggo": "kevin" }, { "id": 1, "doggo": "kevin" },
{ "id": 2, "doggo": { "name": "bob", "age": 20 } }, { "id": 2, "doggo": { "name": "bob", "age": 20 } },
{ "id": 3, "name": "jean", "age": 25 }, { "id": 3, "name": "jean", "age": 25 },
]); ]))
.unwrap();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// ensure we get the right real searchable fields + user defined searchable fields // ensure we get the right real searchable fields + user defined searchable fields
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();

29
milli/src/update/clear_documents.rs
View File

@ -82,36 +82,25 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use heed::EnvOpenOptions;
use super::*; use super::*;
use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig}; use crate::index::tests::TempIndex;
#[test] #[test]
fn clear_documents() { fn clear_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let content = documents!([ index
{ "id": 0, "name": "kevin", "age": 20 }, .add_documents_using_wtxn(&mut wtxn, documents!([
{ "id": 1, "name": "kevina" }, { "id": 0, "name": "kevin", "age": 20 },
{ "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } } { "id": 1, "name": "kevina" },
]); { "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } }
let indexing_config = IndexDocumentsConfig::default(); ]))
let config = IndexerConfig::default(); .unwrap();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
// Clear all documents from the database. // Clear all documents from the database.
let builder = ClearDocuments::new(&mut wtxn, &index); let builder = ClearDocuments::new(&mut wtxn, &index);
assert_eq!(builder.execute().unwrap(), 3); assert_eq!(builder.execute().unwrap(), 3);
wtxn.commit().unwrap(); wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();

376
milli/src/update/delete_documents.rs
View File

@ -654,26 +654,13 @@ where
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use big_s::S; use big_s::S;
use heed::{EnvOpenOptions, RwTxn}; use heed::RwTxn;
use maplit::hashset; use maplit::hashset;
use super::*; use super::*;
use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings}; use crate::index::tests::TempIndex;
use crate::Filter; use crate::Filter;
fn insert_documents<'t, R: std::io::Read + std::io::Seek>(
wtxn: &mut RwTxn<'t, '_>,
index: &'t Index,
documents: crate::documents::DocumentsBatchReader<R>,
) {
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder = IndexDocuments::new(wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap();
}
fn delete_documents<'t>( fn delete_documents<'t>(
wtxn: &mut RwTxn<'t, '_>, wtxn: &mut RwTxn<'t, '_>,
index: &'t Index, index: &'t Index,
@ -695,24 +682,19 @@ mod tests {
#[test] #[test]
fn delete_documents_with_numbers_as_primary_key() { fn delete_documents_with_numbers_as_primary_key() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let content = documents!([ index
{ "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } }, .add_documents_using_wtxn(
{ "id": 1, "name": "kevina", "array": ["I", "am", "fine"] }, &mut wtxn,
{ "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] } documents!([
]); { "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } },
let config = IndexerConfig::default(); { "id": 1, "name": "kevina", "array": ["I", "am", "fine"] },
let indexing_config = IndexDocumentsConfig::default(); { "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] }
let builder = ]),
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); )
let (builder, user_error) = builder.add_documents(content).unwrap(); .unwrap();
user_error.unwrap();
builder.execute().unwrap();
// delete those documents, ids are synchronous therefore 0, 1, and 2. // delete those documents, ids are synchronous therefore 0, 1, and 2.
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
@ -730,25 +712,19 @@ mod tests {
#[test] #[test]
fn delete_documents_with_strange_primary_key() { fn delete_documents_with_strange_primary_key() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let content = documents!([ index
{ "mysuperid": 0, "name": "kevin" }, .add_documents_using_wtxn(
{ "mysuperid": 1, "name": "kevina" }, &mut wtxn,
{ "mysuperid": 2, "name": "benoit" } documents!([
]); { "mysuperid": 0, "name": "kevin" },
{ "mysuperid": 1, "name": "kevina" },
let config = IndexerConfig::default(); { "mysuperid": 2, "name": "benoit" }
let indexing_config = IndexDocumentsConfig::default(); ]),
let builder = )
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); .unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
// Delete not all of the documents but some of them. // Delete not all of the documents but some of them.
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
@ -761,42 +737,45 @@ mod tests {
#[test] #[test]
fn filtered_placeholder_search_should_not_return_deleted_documents() { fn filtered_placeholder_search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("docid"));
builder.set_filterable_fields(hashset! { S("label") });
builder.execute(|_| ()).unwrap();
let content = documents!([ index
{ "docid": "1_4", "label": "sign" }, .update_settings_using_wtxn(&mut wtxn, |settings| {
{ "docid": "1_5", "label": "letter" }, settings.set_primary_key(S("docid"));
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, settings.set_filterable_fields(hashset! { S("label") });
{ "docid": "1_36", "label": "drawing,painting,pattern" }, })
{ "docid": "1_37", "label": "art,drawing,outdoor" }, .unwrap();
{ "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_39", "label": "abstract" }, index
{ "docid": "1_40", "label": "cartoon" }, .add_documents_using_wtxn(
{ "docid": "1_41", "label": "art,drawing" }, &mut wtxn,
{ "docid": "1_42", "label": "art,pattern" }, documents!([
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" }, { "docid": "1_4", "label": "sign" },
{ "docid": "1_44", "label": "drawing" }, { "docid": "1_5", "label": "letter" },
{ "docid": "1_45", "label": "art" }, { "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" }, { "docid": "1_36", "label": "drawing,painting,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" }, { "docid": "1_37", "label": "art,drawing,outdoor" },
{ "docid": "1_52", "label": "abstract,cartoon" }, { "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" }, { "docid": "1_39", "label": "abstract" },
{ "docid": "1_58", "label": "abstract,art,cartoon" }, { "docid": "1_40", "label": "cartoon" },
{ "docid": "1_68", "label": "design" }, { "docid": "1_41", "label": "art,drawing" },
{ "docid": "1_69", "label": "geometry" } { "docid": "1_42", "label": "art,pattern" },
]); { "docid": "1_43", "label": "abstract,art,drawing,pattern" },
{ "docid": "1_44", "label": "drawing" },
{ "docid": "1_45", "label": "art" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" },
{ "docid": "1_52", "label": "abstract,cartoon" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
{ "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
delete_documents(&mut wtxn, &index, &["1_4"]); delete_documents(&mut wtxn, &index, &["1_4"]);
// Placeholder search with filter // Placeholder search with filter
@ -809,41 +788,43 @@ mod tests {
#[test] #[test]
fn placeholder_search_should_not_return_deleted_documents() { fn placeholder_search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); index
let mut builder = Settings::new(&mut wtxn, &index, &config); .update_settings_using_wtxn(&mut wtxn, |settings| {
builder.set_primary_key(S("docid")); settings.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap(); })
.unwrap();
let content = documents!([ index
{ "docid": "1_4", "label": "sign" }, .add_documents_using_wtxn(
{ "docid": "1_5", "label": "letter" }, &mut wtxn,
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, documents!([
{ "docid": "1_36", "label": "drawing,painting,pattern" }, { "docid": "1_4", "label": "sign" },
{ "docid": "1_37", "label": "art,drawing,outdoor" }, { "docid": "1_5", "label": "letter" },
{ "docid": "1_38", "label": "aquarium,art,drawing" }, { "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
{ "docid": "1_39", "label": "abstract" }, { "docid": "1_36", "label": "drawing,painting,pattern" },
{ "docid": "1_40", "label": "cartoon" }, { "docid": "1_37", "label": "art,drawing,outdoor" },
{ "docid": "1_41", "label": "art,drawing" }, { "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_42", "label": "art,pattern" }, { "docid": "1_39", "label": "abstract" },
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" }, { "docid": "1_40", "label": "cartoon" },
{ "docid": "1_44", "label": "drawing" }, { "docid": "1_41", "label": "art,drawing" },
{ "docid": "1_45", "label": "art" }, { "docid": "1_42", "label": "art,pattern" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" }, { "docid": "1_43", "label": "abstract,art,drawing,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" }, { "docid": "1_44", "label": "drawing" },
{ "docid": "1_52", "label": "abstract,cartoon" }, { "docid": "1_45", "label": "art" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" }, { "docid": "1_46", "label": "abstract,colorfulness,pattern" },
{ "docid": "1_58", "label": "abstract,art,cartoon" }, { "docid": "1_47", "label": "abstract,pattern" },
{ "docid": "1_68", "label": "design" }, { "docid": "1_52", "label": "abstract,cartoon" },
{ "docid": "1_69", "label": "geometry" } { "docid": "1_57", "label": "abstract,drawing,pattern" },
]); { "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_4"]); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_4"]);
// Placeholder search // Placeholder search
@ -862,41 +843,43 @@ mod tests {
#[test] #[test]
fn search_should_not_return_deleted_documents() { fn search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); index
let mut builder = Settings::new(&mut wtxn, &index, &config); .update_settings_using_wtxn(&mut wtxn, |settings| {
builder.set_primary_key(S("docid")); settings.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap(); })
.unwrap();
let content = documents!([ index
{"docid": "1_4", "label": "sign"}, .add_documents_using_wtxn(
{"docid": "1_5", "label": "letter"}, &mut wtxn,
{"docid": "1_7", "label": "abstract,cartoon,design,pattern"}, documents!([
{"docid": "1_36","label": "drawing,painting,pattern"}, {"docid": "1_4", "label": "sign"},
{"docid": "1_37","label": "art,drawing,outdoor"}, {"docid": "1_5", "label": "letter"},
{"docid": "1_38","label": "aquarium,art,drawing"}, {"docid": "1_7", "label": "abstract,cartoon,design,pattern"},
{"docid": "1_39","label": "abstract"}, {"docid": "1_36","label": "drawing,painting,pattern"},
{"docid": "1_40","label": "cartoon"}, {"docid": "1_37","label": "art,drawing,outdoor"},
{"docid": "1_41","label": "art,drawing"}, {"docid": "1_38","label": "aquarium,art,drawing"},
{"docid": "1_42","label": "art,pattern"}, {"docid": "1_39","label": "abstract"},
{"docid": "1_43","label": "abstract,art,drawing,pattern"}, {"docid": "1_40","label": "cartoon"},
{"docid": "1_44","label": "drawing"}, {"docid": "1_41","label": "art,drawing"},
{"docid": "1_45","label": "art"}, {"docid": "1_42","label": "art,pattern"},
{"docid": "1_46","label": "abstract,colorfulness,pattern"}, {"docid": "1_43","label": "abstract,art,drawing,pattern"},
{"docid": "1_47","label": "abstract,pattern"}, {"docid": "1_44","label": "drawing"},
{"docid": "1_52","label": "abstract,cartoon"}, {"docid": "1_45","label": "art"},
{"docid": "1_57","label": "abstract,drawing,pattern"}, {"docid": "1_46","label": "abstract,colorfulness,pattern"},
{"docid": "1_58","label": "abstract,art,cartoon"}, {"docid": "1_47","label": "abstract,pattern"},
{"docid": "1_68","label": "design"}, {"docid": "1_52","label": "abstract,cartoon"},
{"docid": "1_69","label": "geometry"} {"docid": "1_57","label": "abstract,drawing,pattern"},
]); {"docid": "1_58","label": "abstract,art,cartoon"},
{"docid": "1_68","label": "design"},
{"docid": "1_69","label": "geometry"}
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_7", "1_52"]); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_7", "1_52"]);
// search for abstract // search for abstract
@ -915,20 +898,18 @@ mod tests {
#[test] #[test]
fn geo_filtered_placeholder_search_should_not_return_deleted_documents() { fn geo_filtered_placeholder_search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); index
let mut builder = Settings::new(&mut wtxn, &index, &config); .update_settings_using_wtxn(&mut wtxn, |settings| {
builder.set_primary_key(S("id")); settings.set_primary_key(S("id"));
builder.set_filterable_fields(hashset!(S("_geo"))); settings.set_filterable_fields(hashset!(S("_geo")));
builder.set_sortable_fields(hashset!(S("_geo"))); settings.set_sortable_fields(hashset!(S("_geo")));
builder.execute(|_| ()).unwrap(); })
.unwrap();
let content = documents!([ index.add_documents_using_wtxn(&mut wtxn, documents!([
{ "id": "1", "city": "Lille", "_geo": { "lat": 50.6299, "lng": 3.0569 } }, { "id": "1", "city": "Lille", "_geo": { "lat": 50.6299, "lng": 3.0569 } },
{ "id": "2", "city": "Mons-en-Barœul", "_geo": { "lat": 50.6415, "lng": 3.1106 } }, { "id": "2", "city": "Mons-en-Barœul", "_geo": { "lat": 50.6415, "lng": 3.1106 } },
{ "id": "3", "city": "Hellemmes", "_geo": { "lat": 50.6312, "lng": 3.1106 } }, { "id": "3", "city": "Hellemmes", "_geo": { "lat": 50.6312, "lng": 3.1106 } },
@ -949,10 +930,9 @@ mod tests {
{ "id": "18", "city": "Amiens", "_geo": { "lat": 49.9314, "lng": 2.2710 } }, { "id": "18", "city": "Amiens", "_geo": { "lat": 49.9314, "lng": 2.2710 } },
{ "id": "19", "city": "Compiègne", "_geo": { "lat": 49.4449, "lng": 2.7913 } }, { "id": "19", "city": "Compiègne", "_geo": { "lat": 49.4449, "lng": 2.7913 } },
{ "id": "20", "city": "Paris", "_geo": { "lat": 48.9021, "lng": 2.3708 } } { "id": "20", "city": "Paris", "_geo": { "lat": 48.9021, "lng": 2.3708 } }
]); ])).unwrap();
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
insert_documents(&mut wtxn, &index, content); let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &external_ids_to_delete); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &external_ids_to_delete);
// Placeholder search with geo filter // Placeholder search with geo filter
@ -972,41 +952,43 @@ mod tests {
#[test] #[test]
fn get_documents_should_not_return_deleted_documents() { fn get_documents_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); index
let mut builder = Settings::new(&mut wtxn, &index, &config); .update_settings_using_wtxn(&mut wtxn, |settings| {
builder.set_primary_key(S("docid")); settings.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap(); })
.unwrap();
let content = documents!([ index
{ "docid": "1_4", "label": "sign" }, .add_documents_using_wtxn(
{ "docid": "1_5", "label": "letter" }, &mut wtxn,
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, documents!([
{ "docid": "1_36", "label": "drawing,painting,pattern" }, { "docid": "1_4", "label": "sign" },
{ "docid": "1_37", "label": "art,drawing,outdoor" }, { "docid": "1_5", "label": "letter" },
{ "docid": "1_38", "label": "aquarium,art,drawing" }, { "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
{ "docid": "1_39", "label": "abstract" }, { "docid": "1_36", "label": "drawing,painting,pattern" },
{ "docid": "1_40", "label": "cartoon" }, { "docid": "1_37", "label": "art,drawing,outdoor" },
{ "docid": "1_41", "label": "art,drawing" }, { "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_42", "label": "art,pattern" }, { "docid": "1_39", "label": "abstract" },
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" }, { "docid": "1_40", "label": "cartoon" },
{ "docid": "1_44", "label": "drawing" }, { "docid": "1_41", "label": "art,drawing" },
{ "docid": "1_45", "label": "art" }, { "docid": "1_42", "label": "art,pattern" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" }, { "docid": "1_43", "label": "abstract,art,drawing,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" }, { "docid": "1_44", "label": "drawing" },
{ "docid": "1_52", "label": "abstract,cartoon" }, { "docid": "1_45", "label": "art" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" }, { "docid": "1_46", "label": "abstract,colorfulness,pattern" },
{ "docid": "1_58", "label": "abstract,art,cartoon" }, { "docid": "1_47", "label": "abstract,pattern" },
{ "docid": "1_68", "label": "design" }, { "docid": "1_52", "label": "abstract,cartoon" },
{ "docid": "1_69", "label": "geometry" } { "docid": "1_57", "label": "abstract,drawing,pattern" },
]); { "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
let deleted_external_ids = ["1_7", "1_52"]; let deleted_external_ids = ["1_7", "1_52"];
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &deleted_external_ids); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &deleted_external_ids);
@ -1042,18 +1024,17 @@ mod tests {
#[test] #[test]
fn stats_should_not_return_deleted_documents() { fn stats_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap();
let content = documents!([ index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("docid"));
})
.unwrap();
index.add_documents_using_wtxn(&mut wtxn, documents!([
{ "docid": "1_4", "label": "sign"}, { "docid": "1_4", "label": "sign"},
{ "docid": "1_5", "label": "letter"}, { "docid": "1_5", "label": "letter"},
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern", "title": "Mickey Mouse"}, { "docid": "1_7", "label": "abstract,cartoon,design,pattern", "title": "Mickey Mouse"},
@ -1074,9 +1055,8 @@ mod tests {
{ "docid": "1_58", "label": "abstract,art,cartoon"}, { "docid": "1_58", "label": "abstract,art,cartoon"},
{ "docid": "1_68", "label": "design"}, { "docid": "1_68", "label": "design"},
{ "docid": "1_69", "label": "geometry"} { "docid": "1_69", "label": "geometry"}
]); ])).unwrap();
insert_documents(&mut wtxn, &index, content);
delete_documents(&mut wtxn, &index, &["1_7", "1_52"]); delete_documents(&mut wtxn, &index, &["1_7", "1_52"]);
// count internal documents // count internal documents

1328
milli/src/update/index_documents/mod.rs
View File

File diff suppressed because it is too large Load Diff

823
milli/src/update/settings.rs
View File

File diff suppressed because it is too large Load Diff