mirror of
https://github.com/meilisearch/meilisearch.git
synced 2024-11-23 10:37:41 +08:00
Simplify indexing tests
This commit is contained in:
Loïc Lecrenier
parent
21284cf235
commit
acff17fb88
@ -179,6 +179,17 @@ macro_rules! documents {
|
||||
}};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn documents_batch_reader_from_objects(
|
||||
objects: impl IntoIterator<Item = Object>,
|
||||
) -> DocumentsBatchReader<std::io::Cursor<Vec<u8>>> {
|
||||
let mut builder = DocumentsBatchBuilder::new(Vec::new());
|
||||
for object in objects {
|
||||
builder.append_json_object(&object).unwrap();
|
||||
}
|
||||
DocumentsBatchReader::from_reader(std::io::Cursor::new(builder.into_inner().unwrap())).unwrap()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::io::Cursor;
|
||||
|
||||
@ -1182,16 +1182,19 @@ pub(crate) mod tests {
|
||||
use std::ops::Deref;
|
||||
|
||||
use big_s::S;
|
||||
use heed::EnvOpenOptions;
|
||||
use heed::{EnvOpenOptions, RwTxn};
|
||||
use maplit::btreemap;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use crate::documents::DocumentsBatchReader;
|
||||
use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS};
|
||||
use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig};
|
||||
use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
|
||||
use crate::Index;
|
||||
|
||||
pub(crate) struct TempIndex {
|
||||
inner: Index,
|
||||
pub inner: Index,
|
||||
pub indexer_config: IndexerConfig,
|
||||
pub index_documents_config: IndexDocumentsConfig,
|
||||
_tempdir: TempDir,
|
||||
}
|
||||
|
||||
@ -1204,43 +1207,88 @@ pub(crate) mod tests {
|
||||
}
|
||||
|
||||
impl TempIndex {
|
||||
/// Creates a temporary index, with a default `4096 * 100` size. This should be enough for
|
||||
/// most tests.
|
||||
pub fn new() -> Self {
|
||||
/// Creates a temporary index
|
||||
pub fn new_with_map_size(size: usize) -> Self {
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(100 * 4096);
|
||||
options.map_size(size);
|
||||
let _tempdir = TempDir::new_in(".").unwrap();
|
||||
let inner = Index::new(options, _tempdir.path()).unwrap();
|
||||
Self { inner, _tempdir }
|
||||
let indexer_config = IndexerConfig::default();
|
||||
let index_documents_config = IndexDocumentsConfig::default();
|
||||
Self { inner, indexer_config, index_documents_config, _tempdir }
|
||||
}
|
||||
/// Creates a temporary index, with a default `4096 * 1000` size. This should be enough for
|
||||
/// most tests.
|
||||
pub fn new() -> Self {
|
||||
Self::new_with_map_size(4096 * 1000)
|
||||
}
|
||||
pub fn add_documents_using_wtxn<'t, R>(
|
||||
&'t self,
|
||||
wtxn: &mut RwTxn<'t, '_>,
|
||||
documents: DocumentsBatchReader<R>,
|
||||
) -> Result<(), crate::error::Error>
|
||||
where
|
||||
R: std::io::Read + std::io::Seek,
|
||||
{
|
||||
let builder = IndexDocuments::new(
|
||||
wtxn,
|
||||
&self,
|
||||
&self.indexer_config,
|
||||
self.index_documents_config.clone(),
|
||||
|_| (),
|
||||
)
|
||||
.unwrap();
|
||||
let (builder, user_error) = builder.add_documents(documents).unwrap();
|
||||
user_error?;
|
||||
builder.execute()?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn add_documents<R>(
|
||||
&self,
|
||||
documents: DocumentsBatchReader<R>,
|
||||
) -> Result<(), crate::error::Error>
|
||||
where
|
||||
R: std::io::Read + std::io::Seek,
|
||||
{
|
||||
let mut wtxn = self.write_txn().unwrap();
|
||||
self.add_documents_using_wtxn(&mut wtxn, documents)?;
|
||||
wtxn.commit().unwrap();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn update_settings(
|
||||
&self,
|
||||
update: impl Fn(&mut Settings),
|
||||
) -> Result<(), crate::error::Error> {
|
||||
let mut wtxn = self.write_txn().unwrap();
|
||||
self.update_settings_using_wtxn(&mut wtxn, update)?;
|
||||
wtxn.commit().unwrap();
|
||||
Ok(())
|
||||
}
|
||||
pub fn update_settings_using_wtxn<'t>(
|
||||
&'t self,
|
||||
wtxn: &mut RwTxn<'t, '_>,
|
||||
update: impl Fn(&mut Settings),
|
||||
) -> Result<(), crate::error::Error> {
|
||||
let mut builder = update::Settings::new(wtxn, &self.inner, &self.indexer_config);
|
||||
update(&mut builder);
|
||||
builder.execute(drop)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn initial_field_distribution() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let content = documents!([
|
||||
{ "id": 1, "name": "kevin" },
|
||||
{ "id": 2, "name": "bob", "age": 20 },
|
||||
{ "id": 2, "name": "bob", "age": 20 },
|
||||
]);
|
||||
|
||||
let config = IndexerConfig::default();
|
||||
let indexing_config = IndexDocumentsConfig::default();
|
||||
let builder =
|
||||
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
|
||||
.unwrap();
|
||||
let (builder, user_error) = builder.add_documents(content).unwrap();
|
||||
user_error.unwrap();
|
||||
builder.execute().unwrap();
|
||||
wtxn.commit().unwrap();
|
||||
let index = TempIndex::new();
|
||||
index
|
||||
.add_documents(documents!([
|
||||
{ "id": 1, "name": "kevin" },
|
||||
{ "id": 2, "name": "bob", "age": 20 },
|
||||
{ "id": 2, "name": "bob", "age": 20 },
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let rtxn = index.read_txn().unwrap();
|
||||
|
||||
let field_distribution = index.field_distribution(&rtxn).unwrap();
|
||||
assert_eq!(
|
||||
field_distribution,
|
||||
@ -1253,19 +1301,13 @@ pub(crate) mod tests {
|
||||
|
||||
// we add all the documents a second time. we are supposed to get the same
|
||||
// field_distribution in the end
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let builder =
|
||||
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
|
||||
.unwrap();
|
||||
let content = documents!([
|
||||
{ "id": 1, "name": "kevin" },
|
||||
{ "id": 2, "name": "bob", "age": 20 },
|
||||
{ "id": 2, "name": "bob", "age": 20 },
|
||||
]);
|
||||
let (builder, user_error) = builder.add_documents(content).unwrap();
|
||||
user_error.unwrap();
|
||||
builder.execute().unwrap();
|
||||
wtxn.commit().unwrap();
|
||||
index
|
||||
.add_documents(documents!([
|
||||
{ "id": 1, "name": "kevin" },
|
||||
{ "id": 2, "name": "bob", "age": 20 },
|
||||
{ "id": 2, "name": "bob", "age": 20 },
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let rtxn = index.read_txn().unwrap();
|
||||
|
||||
@ -1280,19 +1322,12 @@ pub(crate) mod tests {
|
||||
);
|
||||
|
||||
// then we update a document by removing one field and another by adding one field
|
||||
let content = documents!([
|
||||
{ "id": 1, "name": "kevin", "has_dog": true },
|
||||
{ "id": 2, "name": "bob" }
|
||||
]);
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let builder =
|
||||
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
|
||||
.unwrap();
|
||||
let (builder, user_error) = builder.add_documents(content).unwrap();
|
||||
user_error.unwrap();
|
||||
builder.execute().unwrap();
|
||||
wtxn.commit().unwrap();
|
||||
index
|
||||
.add_documents(documents!([
|
||||
{ "id": 1, "name": "kevin", "has_dog": true },
|
||||
{ "id": 2, "name": "bob" }
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
let rtxn = index.read_txn().unwrap();
|
||||
|
||||
@ -1341,35 +1376,19 @@ pub(crate) mod tests {
|
||||
|
||||
#[test]
|
||||
fn add_documents_and_set_searchable_fields() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let content = documents!([
|
||||
{ "id": 1, "doggo": "kevin" },
|
||||
{ "id": 2, "doggo": { "name": "bob", "age": 20 } },
|
||||
{ "id": 3, "name": "jean", "age": 25 },
|
||||
]);
|
||||
|
||||
let config = IndexerConfig::default();
|
||||
let indexing_config = IndexDocumentsConfig::default();
|
||||
let builder =
|
||||
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
|
||||
.unwrap();
|
||||
let (builder, user_error) = builder.add_documents(content).unwrap();
|
||||
user_error.unwrap();
|
||||
builder.execute().unwrap();
|
||||
wtxn.commit().unwrap();
|
||||
|
||||
// set searchable fields
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let mut builder = update::Settings::new(&mut wtxn, &index, &config);
|
||||
builder.set_searchable_fields(vec![S("doggo"), S("name")]);
|
||||
|
||||
builder.execute(drop).unwrap();
|
||||
wtxn.commit().unwrap();
|
||||
let index = TempIndex::new();
|
||||
index
|
||||
.add_documents(documents!([
|
||||
{ "id": 1, "doggo": "kevin" },
|
||||
{ "id": 2, "doggo": { "name": "bob", "age": 20 } },
|
||||
{ "id": 3, "name": "jean", "age": 25 },
|
||||
]))
|
||||
.unwrap();
|
||||
index
|
||||
.update_settings(|settings| {
|
||||
settings.set_searchable_fields(vec![S("doggo"), S("name")]);
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// ensure we get the right real searchable fields + user defined searchable fields
|
||||
let rtxn = index.read_txn().unwrap();
|
||||
@ -1383,19 +1402,13 @@ pub(crate) mod tests {
|
||||
|
||||
#[test]
|
||||
fn set_searchable_fields_and_add_documents() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let config = IndexerConfig::default();
|
||||
let index = TempIndex::new();
|
||||
|
||||
// set searchable fields
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let mut builder = update::Settings::new(&mut wtxn, &index, &config);
|
||||
builder.set_searchable_fields(vec![S("doggo"), S("name")]);
|
||||
|
||||
builder.execute(drop).unwrap();
|
||||
wtxn.commit().unwrap();
|
||||
index
|
||||
.update_settings(|settings| {
|
||||
settings.set_searchable_fields(vec![S("doggo"), S("name")]);
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// ensure we get the right real searchable fields + user defined searchable fields
|
||||
let rtxn = index.read_txn().unwrap();
|
||||
@ -1405,21 +1418,13 @@ pub(crate) mod tests {
|
||||
let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap();
|
||||
assert_eq!(user_defined, &["doggo", "name"]);
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let content = documents!([
|
||||
{ "id": 1, "doggo": "kevin" },
|
||||
{ "id": 2, "doggo": { "name": "bob", "age": 20 } },
|
||||
{ "id": 3, "name": "jean", "age": 25 },
|
||||
]);
|
||||
|
||||
let indexing_config = IndexDocumentsConfig::default();
|
||||
let builder =
|
||||
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
|
||||
.unwrap();
|
||||
let (builder, user_error) = builder.add_documents(content).unwrap();
|
||||
user_error.unwrap();
|
||||
builder.execute().unwrap();
|
||||
wtxn.commit().unwrap();
|
||||
index
|
||||
.add_documents(documents!([
|
||||
{ "id": 1, "doggo": "kevin" },
|
||||
{ "id": 2, "doggo": { "name": "bob", "age": 20 } },
|
||||
{ "id": 3, "name": "jean", "age": 25 },
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
// ensure we get the right real searchable fields + user defined searchable fields
|
||||
let rtxn = index.read_txn().unwrap();
|
||||
|
||||
@ -82,36 +82,25 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use heed::EnvOpenOptions;
|
||||
|
||||
use super::*;
|
||||
use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig};
|
||||
use crate::index::tests::TempIndex;
|
||||
|
||||
#[test]
|
||||
fn clear_documents() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let index = TempIndex::new();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let content = documents!([
|
||||
{ "id": 0, "name": "kevin", "age": 20 },
|
||||
{ "id": 1, "name": "kevina" },
|
||||
{ "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } }
|
||||
]);
|
||||
let indexing_config = IndexDocumentsConfig::default();
|
||||
let config = IndexerConfig::default();
|
||||
let builder =
|
||||
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
|
||||
let (builder, user_error) = builder.add_documents(content).unwrap();
|
||||
user_error.unwrap();
|
||||
builder.execute().unwrap();
|
||||
index
|
||||
.add_documents_using_wtxn(&mut wtxn, documents!([
|
||||
{ "id": 0, "name": "kevin", "age": 20 },
|
||||
{ "id": 1, "name": "kevina" },
|
||||
{ "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } }
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
// Clear all documents from the database.
|
||||
let builder = ClearDocuments::new(&mut wtxn, &index);
|
||||
assert_eq!(builder.execute().unwrap(), 3);
|
||||
|
||||
wtxn.commit().unwrap();
|
||||
|
||||
let rtxn = index.read_txn().unwrap();
|
||||
|
||||
@ -654,26 +654,13 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use big_s::S;
|
||||
use heed::{EnvOpenOptions, RwTxn};
|
||||
use heed::RwTxn;
|
||||
use maplit::hashset;
|
||||
|
||||
use super::*;
|
||||
use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
|
||||
use crate::index::tests::TempIndex;
|
||||
use crate::Filter;
|
||||
|
||||
fn insert_documents<'t, R: std::io::Read + std::io::Seek>(
|
||||
wtxn: &mut RwTxn<'t, '_>,
|
||||
index: &'t Index,
|
||||
documents: crate::documents::DocumentsBatchReader<R>,
|
||||
) {
|
||||
let config = IndexerConfig::default();
|
||||
let indexing_config = IndexDocumentsConfig::default();
|
||||
let builder = IndexDocuments::new(wtxn, &index, &config, indexing_config, |_| ()).unwrap();
|
||||
let (builder, user_error) = builder.add_documents(documents).unwrap();
|
||||
user_error.unwrap();
|
||||
builder.execute().unwrap();
|
||||
}
|
||||
|
||||
fn delete_documents<'t>(
|
||||
wtxn: &mut RwTxn<'t, '_>,
|
||||
index: &'t Index,
|
||||
@ -695,24 +682,19 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn delete_documents_with_numbers_as_primary_key() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let index = TempIndex::new();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let content = documents!([
|
||||
{ "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } },
|
||||
{ "id": 1, "name": "kevina", "array": ["I", "am", "fine"] },
|
||||
{ "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] }
|
||||
]);
|
||||
let config = IndexerConfig::default();
|
||||
let indexing_config = IndexDocumentsConfig::default();
|
||||
let builder =
|
||||
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
|
||||
let (builder, user_error) = builder.add_documents(content).unwrap();
|
||||
user_error.unwrap();
|
||||
builder.execute().unwrap();
|
||||
index
|
||||
.add_documents_using_wtxn(
|
||||
&mut wtxn,
|
||||
documents!([
|
||||
{ "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } },
|
||||
{ "id": 1, "name": "kevina", "array": ["I", "am", "fine"] },
|
||||
{ "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] }
|
||||
]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// delete those documents, ids are synchronous therefore 0, 1, and 2.
|
||||
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
|
||||
@ -730,25 +712,19 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn delete_documents_with_strange_primary_key() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let index = TempIndex::new();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let content = documents!([
|
||||
{ "mysuperid": 0, "name": "kevin" },
|
||||
{ "mysuperid": 1, "name": "kevina" },
|
||||
{ "mysuperid": 2, "name": "benoit" }
|
||||
]);
|
||||
|
||||
let config = IndexerConfig::default();
|
||||
let indexing_config = IndexDocumentsConfig::default();
|
||||
let builder =
|
||||
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
|
||||
let (builder, user_error) = builder.add_documents(content).unwrap();
|
||||
user_error.unwrap();
|
||||
builder.execute().unwrap();
|
||||
index
|
||||
.add_documents_using_wtxn(
|
||||
&mut wtxn,
|
||||
documents!([
|
||||
{ "mysuperid": 0, "name": "kevin" },
|
||||
{ "mysuperid": 1, "name": "kevina" },
|
||||
{ "mysuperid": 2, "name": "benoit" }
|
||||
]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Delete not all of the documents but some of them.
|
||||
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
|
||||
@ -761,42 +737,45 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn filtered_placeholder_search_should_not_return_deleted_documents() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let index = TempIndex::new();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let config = IndexerConfig::default();
|
||||
let mut builder = Settings::new(&mut wtxn, &index, &config);
|
||||
builder.set_primary_key(S("docid"));
|
||||
builder.set_filterable_fields(hashset! { S("label") });
|
||||
builder.execute(|_| ()).unwrap();
|
||||
|
||||
let content = documents!([
|
||||
{ "docid": "1_4", "label": "sign" },
|
||||
{ "docid": "1_5", "label": "letter" },
|
||||
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
|
||||
{ "docid": "1_36", "label": "drawing,painting,pattern" },
|
||||
{ "docid": "1_37", "label": "art,drawing,outdoor" },
|
||||
{ "docid": "1_38", "label": "aquarium,art,drawing" },
|
||||
{ "docid": "1_39", "label": "abstract" },
|
||||
{ "docid": "1_40", "label": "cartoon" },
|
||||
{ "docid": "1_41", "label": "art,drawing" },
|
||||
{ "docid": "1_42", "label": "art,pattern" },
|
||||
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" },
|
||||
{ "docid": "1_44", "label": "drawing" },
|
||||
{ "docid": "1_45", "label": "art" },
|
||||
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
|
||||
{ "docid": "1_47", "label": "abstract,pattern" },
|
||||
{ "docid": "1_52", "label": "abstract,cartoon" },
|
||||
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
|
||||
{ "docid": "1_58", "label": "abstract,art,cartoon" },
|
||||
{ "docid": "1_68", "label": "design" },
|
||||
{ "docid": "1_69", "label": "geometry" }
|
||||
]);
|
||||
index
|
||||
.update_settings_using_wtxn(&mut wtxn, |settings| {
|
||||
settings.set_primary_key(S("docid"));
|
||||
settings.set_filterable_fields(hashset! { S("label") });
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
index
|
||||
.add_documents_using_wtxn(
|
||||
&mut wtxn,
|
||||
documents!([
|
||||
{ "docid": "1_4", "label": "sign" },
|
||||
{ "docid": "1_5", "label": "letter" },
|
||||
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
|
||||
{ "docid": "1_36", "label": "drawing,painting,pattern" },
|
||||
{ "docid": "1_37", "label": "art,drawing,outdoor" },
|
||||
{ "docid": "1_38", "label": "aquarium,art,drawing" },
|
||||
{ "docid": "1_39", "label": "abstract" },
|
||||
{ "docid": "1_40", "label": "cartoon" },
|
||||
{ "docid": "1_41", "label": "art,drawing" },
|
||||
{ "docid": "1_42", "label": "art,pattern" },
|
||||
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" },
|
||||
{ "docid": "1_44", "label": "drawing" },
|
||||
{ "docid": "1_45", "label": "art" },
|
||||
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
|
||||
{ "docid": "1_47", "label": "abstract,pattern" },
|
||||
{ "docid": "1_52", "label": "abstract,cartoon" },
|
||||
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
|
||||
{ "docid": "1_58", "label": "abstract,art,cartoon" },
|
||||
{ "docid": "1_68", "label": "design" },
|
||||
{ "docid": "1_69", "label": "geometry" }
|
||||
]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
insert_documents(&mut wtxn, &index, content);
|
||||
delete_documents(&mut wtxn, &index, &["1_4"]);
|
||||
|
||||
// Placeholder search with filter
|
||||
@ -809,41 +788,43 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn placeholder_search_should_not_return_deleted_documents() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let index = TempIndex::new();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let config = IndexerConfig::default();
|
||||
let mut builder = Settings::new(&mut wtxn, &index, &config);
|
||||
builder.set_primary_key(S("docid"));
|
||||
builder.execute(|_| ()).unwrap();
|
||||
index
|
||||
.update_settings_using_wtxn(&mut wtxn, |settings| {
|
||||
settings.set_primary_key(S("docid"));
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let content = documents!([
|
||||
{ "docid": "1_4", "label": "sign" },
|
||||
{ "docid": "1_5", "label": "letter" },
|
||||
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
|
||||
{ "docid": "1_36", "label": "drawing,painting,pattern" },
|
||||
{ "docid": "1_37", "label": "art,drawing,outdoor" },
|
||||
{ "docid": "1_38", "label": "aquarium,art,drawing" },
|
||||
{ "docid": "1_39", "label": "abstract" },
|
||||
{ "docid": "1_40", "label": "cartoon" },
|
||||
{ "docid": "1_41", "label": "art,drawing" },
|
||||
{ "docid": "1_42", "label": "art,pattern" },
|
||||
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" },
|
||||
{ "docid": "1_44", "label": "drawing" },
|
||||
{ "docid": "1_45", "label": "art" },
|
||||
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
|
||||
{ "docid": "1_47", "label": "abstract,pattern" },
|
||||
{ "docid": "1_52", "label": "abstract,cartoon" },
|
||||
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
|
||||
{ "docid": "1_58", "label": "abstract,art,cartoon" },
|
||||
{ "docid": "1_68", "label": "design" },
|
||||
{ "docid": "1_69", "label": "geometry" }
|
||||
]);
|
||||
index
|
||||
.add_documents_using_wtxn(
|
||||
&mut wtxn,
|
||||
documents!([
|
||||
{ "docid": "1_4", "label": "sign" },
|
||||
{ "docid": "1_5", "label": "letter" },
|
||||
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
|
||||
{ "docid": "1_36", "label": "drawing,painting,pattern" },
|
||||
{ "docid": "1_37", "label": "art,drawing,outdoor" },
|
||||
{ "docid": "1_38", "label": "aquarium,art,drawing" },
|
||||
{ "docid": "1_39", "label": "abstract" },
|
||||
{ "docid": "1_40", "label": "cartoon" },
|
||||
{ "docid": "1_41", "label": "art,drawing" },
|
||||
{ "docid": "1_42", "label": "art,pattern" },
|
||||
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" },
|
||||
{ "docid": "1_44", "label": "drawing" },
|
||||
{ "docid": "1_45", "label": "art" },
|
||||
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
|
||||
{ "docid": "1_47", "label": "abstract,pattern" },
|
||||
{ "docid": "1_52", "label": "abstract,cartoon" },
|
||||
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
|
||||
{ "docid": "1_58", "label": "abstract,art,cartoon" },
|
||||
{ "docid": "1_68", "label": "design" },
|
||||
{ "docid": "1_69", "label": "geometry" }
|
||||
]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
insert_documents(&mut wtxn, &index, content);
|
||||
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_4"]);
|
||||
|
||||
// Placeholder search
|
||||
@ -862,41 +843,43 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn search_should_not_return_deleted_documents() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let index = TempIndex::new();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let config = IndexerConfig::default();
|
||||
let mut builder = Settings::new(&mut wtxn, &index, &config);
|
||||
builder.set_primary_key(S("docid"));
|
||||
builder.execute(|_| ()).unwrap();
|
||||
index
|
||||
.update_settings_using_wtxn(&mut wtxn, |settings| {
|
||||
settings.set_primary_key(S("docid"));
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let content = documents!([
|
||||
{"docid": "1_4", "label": "sign"},
|
||||
{"docid": "1_5", "label": "letter"},
|
||||
{"docid": "1_7", "label": "abstract,cartoon,design,pattern"},
|
||||
{"docid": "1_36","label": "drawing,painting,pattern"},
|
||||
{"docid": "1_37","label": "art,drawing,outdoor"},
|
||||
{"docid": "1_38","label": "aquarium,art,drawing"},
|
||||
{"docid": "1_39","label": "abstract"},
|
||||
{"docid": "1_40","label": "cartoon"},
|
||||
{"docid": "1_41","label": "art,drawing"},
|
||||
{"docid": "1_42","label": "art,pattern"},
|
||||
{"docid": "1_43","label": "abstract,art,drawing,pattern"},
|
||||
{"docid": "1_44","label": "drawing"},
|
||||
{"docid": "1_45","label": "art"},
|
||||
{"docid": "1_46","label": "abstract,colorfulness,pattern"},
|
||||
{"docid": "1_47","label": "abstract,pattern"},
|
||||
{"docid": "1_52","label": "abstract,cartoon"},
|
||||
{"docid": "1_57","label": "abstract,drawing,pattern"},
|
||||
{"docid": "1_58","label": "abstract,art,cartoon"},
|
||||
{"docid": "1_68","label": "design"},
|
||||
{"docid": "1_69","label": "geometry"}
|
||||
]);
|
||||
index
|
||||
.add_documents_using_wtxn(
|
||||
&mut wtxn,
|
||||
documents!([
|
||||
{"docid": "1_4", "label": "sign"},
|
||||
{"docid": "1_5", "label": "letter"},
|
||||
{"docid": "1_7", "label": "abstract,cartoon,design,pattern"},
|
||||
{"docid": "1_36","label": "drawing,painting,pattern"},
|
||||
{"docid": "1_37","label": "art,drawing,outdoor"},
|
||||
{"docid": "1_38","label": "aquarium,art,drawing"},
|
||||
{"docid": "1_39","label": "abstract"},
|
||||
{"docid": "1_40","label": "cartoon"},
|
||||
{"docid": "1_41","label": "art,drawing"},
|
||||
{"docid": "1_42","label": "art,pattern"},
|
||||
{"docid": "1_43","label": "abstract,art,drawing,pattern"},
|
||||
{"docid": "1_44","label": "drawing"},
|
||||
{"docid": "1_45","label": "art"},
|
||||
{"docid": "1_46","label": "abstract,colorfulness,pattern"},
|
||||
{"docid": "1_47","label": "abstract,pattern"},
|
||||
{"docid": "1_52","label": "abstract,cartoon"},
|
||||
{"docid": "1_57","label": "abstract,drawing,pattern"},
|
||||
{"docid": "1_58","label": "abstract,art,cartoon"},
|
||||
{"docid": "1_68","label": "design"},
|
||||
{"docid": "1_69","label": "geometry"}
|
||||
]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
insert_documents(&mut wtxn, &index, content);
|
||||
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_7", "1_52"]);
|
||||
|
||||
// search for abstract
|
||||
@ -915,20 +898,18 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn geo_filtered_placeholder_search_should_not_return_deleted_documents() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let index = TempIndex::new();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let config = IndexerConfig::default();
|
||||
let mut builder = Settings::new(&mut wtxn, &index, &config);
|
||||
builder.set_primary_key(S("id"));
|
||||
builder.set_filterable_fields(hashset!(S("_geo")));
|
||||
builder.set_sortable_fields(hashset!(S("_geo")));
|
||||
builder.execute(|_| ()).unwrap();
|
||||
index
|
||||
.update_settings_using_wtxn(&mut wtxn, |settings| {
|
||||
settings.set_primary_key(S("id"));
|
||||
settings.set_filterable_fields(hashset!(S("_geo")));
|
||||
settings.set_sortable_fields(hashset!(S("_geo")));
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let content = documents!([
|
||||
index.add_documents_using_wtxn(&mut wtxn, documents!([
|
||||
{ "id": "1", "city": "Lille", "_geo": { "lat": 50.6299, "lng": 3.0569 } },
|
||||
{ "id": "2", "city": "Mons-en-Barœul", "_geo": { "lat": 50.6415, "lng": 3.1106 } },
|
||||
{ "id": "3", "city": "Hellemmes", "_geo": { "lat": 50.6312, "lng": 3.1106 } },
|
||||
@ -949,10 +930,9 @@ mod tests {
|
||||
{ "id": "18", "city": "Amiens", "_geo": { "lat": 49.9314, "lng": 2.2710 } },
|
||||
{ "id": "19", "city": "Compiègne", "_geo": { "lat": 49.4449, "lng": 2.7913 } },
|
||||
{ "id": "20", "city": "Paris", "_geo": { "lat": 48.9021, "lng": 2.3708 } }
|
||||
]);
|
||||
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
|
||||
])).unwrap();
|
||||
|
||||
insert_documents(&mut wtxn, &index, content);
|
||||
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
|
||||
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &external_ids_to_delete);
|
||||
|
||||
// Placeholder search with geo filter
|
||||
@ -972,41 +952,43 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn get_documents_should_not_return_deleted_documents() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let index = TempIndex::new();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let config = IndexerConfig::default();
|
||||
let mut builder = Settings::new(&mut wtxn, &index, &config);
|
||||
builder.set_primary_key(S("docid"));
|
||||
builder.execute(|_| ()).unwrap();
|
||||
index
|
||||
.update_settings_using_wtxn(&mut wtxn, |settings| {
|
||||
settings.set_primary_key(S("docid"));
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let content = documents!([
|
||||
{ "docid": "1_4", "label": "sign" },
|
||||
{ "docid": "1_5", "label": "letter" },
|
||||
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
|
||||
{ "docid": "1_36", "label": "drawing,painting,pattern" },
|
||||
{ "docid": "1_37", "label": "art,drawing,outdoor" },
|
||||
{ "docid": "1_38", "label": "aquarium,art,drawing" },
|
||||
{ "docid": "1_39", "label": "abstract" },
|
||||
{ "docid": "1_40", "label": "cartoon" },
|
||||
{ "docid": "1_41", "label": "art,drawing" },
|
||||
{ "docid": "1_42", "label": "art,pattern" },
|
||||
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" },
|
||||
{ "docid": "1_44", "label": "drawing" },
|
||||
{ "docid": "1_45", "label": "art" },
|
||||
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
|
||||
{ "docid": "1_47", "label": "abstract,pattern" },
|
||||
{ "docid": "1_52", "label": "abstract,cartoon" },
|
||||
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
|
||||
{ "docid": "1_58", "label": "abstract,art,cartoon" },
|
||||
{ "docid": "1_68", "label": "design" },
|
||||
{ "docid": "1_69", "label": "geometry" }
|
||||
]);
|
||||
index
|
||||
.add_documents_using_wtxn(
|
||||
&mut wtxn,
|
||||
documents!([
|
||||
{ "docid": "1_4", "label": "sign" },
|
||||
{ "docid": "1_5", "label": "letter" },
|
||||
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
|
||||
{ "docid": "1_36", "label": "drawing,painting,pattern" },
|
||||
{ "docid": "1_37", "label": "art,drawing,outdoor" },
|
||||
{ "docid": "1_38", "label": "aquarium,art,drawing" },
|
||||
{ "docid": "1_39", "label": "abstract" },
|
||||
{ "docid": "1_40", "label": "cartoon" },
|
||||
{ "docid": "1_41", "label": "art,drawing" },
|
||||
{ "docid": "1_42", "label": "art,pattern" },
|
||||
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" },
|
||||
{ "docid": "1_44", "label": "drawing" },
|
||||
{ "docid": "1_45", "label": "art" },
|
||||
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
|
||||
{ "docid": "1_47", "label": "abstract,pattern" },
|
||||
{ "docid": "1_52", "label": "abstract,cartoon" },
|
||||
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
|
||||
{ "docid": "1_58", "label": "abstract,art,cartoon" },
|
||||
{ "docid": "1_68", "label": "design" },
|
||||
{ "docid": "1_69", "label": "geometry" }
|
||||
]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
insert_documents(&mut wtxn, &index, content);
|
||||
let deleted_external_ids = ["1_7", "1_52"];
|
||||
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &deleted_external_ids);
|
||||
|
||||
@ -1042,18 +1024,17 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn stats_should_not_return_deleted_documents() {
|
||||
let path = tempfile::tempdir().unwrap();
|
||||
let mut options = EnvOpenOptions::new();
|
||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||
let index = Index::new(options, &path).unwrap();
|
||||
let index = TempIndex::new();
|
||||
|
||||
let mut wtxn = index.write_txn().unwrap();
|
||||
let config = IndexerConfig::default();
|
||||
let mut builder = Settings::new(&mut wtxn, &index, &config);
|
||||
builder.set_primary_key(S("docid"));
|
||||
builder.execute(|_| ()).unwrap();
|
||||
|
||||
let content = documents!([
|
||||
index
|
||||
.update_settings_using_wtxn(&mut wtxn, |settings| {
|
||||
settings.set_primary_key(S("docid"));
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
index.add_documents_using_wtxn(&mut wtxn, documents!([
|
||||
{ "docid": "1_4", "label": "sign"},
|
||||
{ "docid": "1_5", "label": "letter"},
|
||||
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern", "title": "Mickey Mouse"},
|
||||
@ -1074,9 +1055,8 @@ mod tests {
|
||||
{ "docid": "1_58", "label": "abstract,art,cartoon"},
|
||||
{ "docid": "1_68", "label": "design"},
|
||||
{ "docid": "1_69", "label": "geometry"}
|
||||
]);
|
||||
])).unwrap();
|
||||
|
||||
insert_documents(&mut wtxn, &index, content);
|
||||
delete_documents(&mut wtxn, &index, &["1_7", "1_52"]);
|
||||
|
||||
// count internal documents
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user