Fix tests after adding DeletionStrategy

This commit is contained in:
Louis Dureuil 2022-12-19 09:47:29 +01:00
parent 171c942282
commit ad9937c755
No known key found for this signature in database
5 changed files with 109 additions and 92 deletions

View File

@ -1192,8 +1192,8 @@ pub(crate) mod tests {
use crate::error::{Error, InternalError};
use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS};
use crate::update::{
self, DeleteDocuments, IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod,
IndexerConfig, Settings,
self, DeleteDocuments, DeletionStrategy, IndexDocuments, IndexDocumentsConfig,
IndexDocumentsMethod, IndexerConfig, Settings,
};
use crate::{db_snap, obkv_to_json, Index};
@ -1282,6 +1282,17 @@ pub(crate) mod tests {
builder.execute(drop, || false)?;
Ok(())
}
pub fn delete_document(&self, external_document_id: &str) {
let mut wtxn = self.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &self).unwrap();
delete.strategy(self.index_documents_config.deletion_strategy);
delete.delete_external_id(external_document_id);
delete.execute().unwrap();
wtxn.commit().unwrap();
}
}
#[test]
@ -1487,7 +1498,9 @@ pub(crate) mod tests {
use big_s::S;
use maplit::hashset;
let index = TempIndex::new();
let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
let index = index;
index
.update_settings(|settings| {
@ -1657,7 +1670,8 @@ pub(crate) mod tests {
}
// Second Batch: replace the documents with soft-deletion
{
index.index_documents_config.disable_soft_deletion = false;
index.index_documents_config.deletion_strategy =
crate::update::DeletionStrategy::AlwaysSoft;
let mut docs1 = vec![];
for i in 0..3 {
docs1.push(serde_json::json!(
@ -1726,7 +1740,7 @@ pub(crate) mod tests {
drop(rtxn);
// Third Batch: replace the documents with soft-deletion again
{
index.index_documents_config.disable_soft_deletion = false;
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
let mut docs1 = vec![];
for i in 0..3 {
docs1.push(serde_json::json!(
@ -1795,7 +1809,7 @@ pub(crate) mod tests {
// Fourth Batch: replace the documents without soft-deletion
{
index.index_documents_config.disable_soft_deletion = true;
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
let mut docs1 = vec![];
for i in 0..3 {
docs1.push(serde_json::json!(
@ -1867,6 +1881,7 @@ pub(crate) mod tests {
fn bug_3021_first() {
// https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index
@ -1891,11 +1906,7 @@ pub(crate) mod tests {
"###);
db_snap!(index, soft_deleted_documents_ids, 1, @"[]");
let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.delete_external_id("34");
delete.execute().unwrap();
wtxn.commit().unwrap();
index.delete_document("34");
db_snap!(index, documents_ids, @"[0, ]");
db_snap!(index, external_documents_ids, 2, @r###"
@ -1936,11 +1947,7 @@ pub(crate) mod tests {
db_snap!(index, soft_deleted_documents_ids, 4, @"[]");
// We do the test again, but deleting the document with id 0 instead of id 1 now
let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.delete_external_id("38");
delete.execute().unwrap();
wtxn.commit().unwrap();
index.delete_document("38");
db_snap!(index, documents_ids, @"[1, ]");
db_snap!(index, external_documents_ids, 5, @r###"
@ -1987,6 +1994,7 @@ pub(crate) mod tests {
fn bug_3021_second() {
// https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
index
@ -2011,11 +2019,7 @@ pub(crate) mod tests {
"###);
db_snap!(index, soft_deleted_documents_ids, 1, @"[]");
let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.delete_external_id("34");
delete.execute().unwrap();
wtxn.commit().unwrap();
index.delete_document("34");
db_snap!(index, documents_ids, @"[0, ]");
db_snap!(index, external_documents_ids, 2, @r###"
@ -2116,6 +2120,7 @@ pub(crate) mod tests {
fn bug_3021_third() {
// https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
index
@ -2142,11 +2147,7 @@ pub(crate) mod tests {
"###);
db_snap!(index, soft_deleted_documents_ids, 1, @"[]");
let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.delete_external_id("3");
delete.execute().unwrap();
wtxn.commit().unwrap();
index.delete_document("3");
db_snap!(index, documents_ids, @"[1, 2, ]");
db_snap!(index, external_documents_ids, 2, @r###"
@ -2158,7 +2159,7 @@ pub(crate) mod tests {
"###);
db_snap!(index, soft_deleted_documents_ids, 2, @"[0, ]");
index.index_documents_config.disable_soft_deletion = true;
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
index.add_documents(documents!([{ "primary_key": "4", "a": 2 }])).unwrap();

View File

@ -685,7 +685,7 @@ mod tests {
wtxn: &mut RwTxn<'t, '_>,
index: &'t Index,
external_ids: &[&str],
disable_soft_deletion: bool,
strategy: DeletionStrategy,
) -> Vec<u32> {
let external_document_ids = index.external_documents_ids(wtxn).unwrap();
let ids_to_delete: Vec<u32> = external_ids
@ -695,14 +695,14 @@ mod tests {
// Delete some documents.
let mut builder = DeleteDocuments::new(wtxn, index).unwrap();
builder.disable_soft_deletion(disable_soft_deletion);
builder.strategy(strategy);
external_ids.iter().for_each(|id| drop(builder.delete_external_id(id)));
builder.execute().unwrap();
ids_to_delete
}
fn delete_documents_with_numbers_as_primary_key_(disable_soft_deletion: bool) {
fn delete_documents_with_numbers_as_primary_key_(deletion_strategy: DeletionStrategy) {
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
@ -722,17 +722,17 @@ mod tests {
builder.delete_document(0);
builder.delete_document(1);
builder.delete_document(2);
builder.disable_soft_deletion(disable_soft_deletion);
builder.strategy(deletion_strategy);
builder.execute().unwrap();
wtxn.commit().unwrap();
// All these snapshots should be empty since the database was cleared
db_snap!(index, documents_ids, disable_soft_deletion);
db_snap!(index, word_docids, disable_soft_deletion);
db_snap!(index, word_pair_proximity_docids, disable_soft_deletion);
db_snap!(index, facet_id_exists_docids, disable_soft_deletion);
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion);
db_snap!(index, documents_ids, deletion_strategy);
db_snap!(index, word_docids, deletion_strategy);
db_snap!(index, word_pair_proximity_docids, deletion_strategy);
db_snap!(index, facet_id_exists_docids, deletion_strategy);
db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
let rtxn = index.read_txn().unwrap();
@ -741,11 +741,11 @@ mod tests {
#[test]
fn delete_documents_with_numbers_as_primary_key() {
delete_documents_with_numbers_as_primary_key_(true);
delete_documents_with_numbers_as_primary_key_(false);
delete_documents_with_numbers_as_primary_key_(DeletionStrategy::AlwaysHard);
delete_documents_with_numbers_as_primary_key_(DeletionStrategy::AlwaysSoft);
}
fn delete_documents_with_strange_primary_key_(disable_soft_deletion: bool) {
fn delete_documents_with_strange_primary_key_(strategy: DeletionStrategy) {
let index = TempIndex::new();
index
@ -771,24 +771,24 @@ mod tests {
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.delete_external_id("0");
builder.delete_external_id("1");
builder.disable_soft_deletion(disable_soft_deletion);
builder.strategy(strategy);
builder.execute().unwrap();
wtxn.commit().unwrap();
db_snap!(index, documents_ids, disable_soft_deletion);
db_snap!(index, word_docids, disable_soft_deletion);
db_snap!(index, word_pair_proximity_docids, disable_soft_deletion);
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion);
db_snap!(index, documents_ids, strategy);
db_snap!(index, word_docids, strategy);
db_snap!(index, word_pair_proximity_docids, strategy);
db_snap!(index, soft_deleted_documents_ids, strategy);
}
#[test]
fn delete_documents_with_strange_primary_key() {
delete_documents_with_strange_primary_key_(true);
delete_documents_with_strange_primary_key_(false);
delete_documents_with_strange_primary_key_(DeletionStrategy::AlwaysHard);
delete_documents_with_strange_primary_key_(DeletionStrategy::AlwaysSoft);
}
fn filtered_placeholder_search_should_not_return_deleted_documents_(
disable_soft_deletion: bool,
deletion_strategy: DeletionStrategy,
) {
let index = TempIndex::new();
@ -832,7 +832,7 @@ mod tests {
)
.unwrap();
delete_documents(&mut wtxn, &index, &["1_4", "1_70", "1_72"], disable_soft_deletion);
delete_documents(&mut wtxn, &index, &["1_4", "1_70", "1_72"], deletion_strategy);
// Placeholder search with filter
let filter = Filter::from_str("label = sign").unwrap().unwrap();
@ -841,21 +841,27 @@ mod tests {
wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion);
db_snap!(index, word_docids, disable_soft_deletion);
db_snap!(index, facet_id_f64_docids, disable_soft_deletion);
db_snap!(index, word_pair_proximity_docids, disable_soft_deletion);
db_snap!(index, facet_id_exists_docids, disable_soft_deletion);
db_snap!(index, facet_id_string_docids, disable_soft_deletion);
db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
db_snap!(index, word_docids, deletion_strategy);
db_snap!(index, facet_id_f64_docids, deletion_strategy);
db_snap!(index, word_pair_proximity_docids, deletion_strategy);
db_snap!(index, facet_id_exists_docids, deletion_strategy);
db_snap!(index, facet_id_string_docids, deletion_strategy);
}
#[test]
fn filtered_placeholder_search_should_not_return_deleted_documents() {
filtered_placeholder_search_should_not_return_deleted_documents_(true);
filtered_placeholder_search_should_not_return_deleted_documents_(false);
filtered_placeholder_search_should_not_return_deleted_documents_(
DeletionStrategy::AlwaysHard,
);
filtered_placeholder_search_should_not_return_deleted_documents_(
DeletionStrategy::AlwaysSoft,
);
}
fn placeholder_search_should_not_return_deleted_documents_(disable_soft_deletion: bool) {
fn placeholder_search_should_not_return_deleted_documents_(
deletion_strategy: DeletionStrategy,
) {
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
@ -896,8 +902,7 @@ mod tests {
)
.unwrap();
let deleted_internal_ids =
delete_documents(&mut wtxn, &index, &["1_4"], disable_soft_deletion);
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_4"], deletion_strategy);
// Placeholder search
let results = index.search(&wtxn).execute().unwrap();
@ -915,11 +920,11 @@ mod tests {
#[test]
fn placeholder_search_should_not_return_deleted_documents() {
placeholder_search_should_not_return_deleted_documents_(true);
placeholder_search_should_not_return_deleted_documents_(false);
placeholder_search_should_not_return_deleted_documents_(DeletionStrategy::AlwaysHard);
placeholder_search_should_not_return_deleted_documents_(DeletionStrategy::AlwaysSoft);
}
fn search_should_not_return_deleted_documents_(disable_soft_deletion: bool) {
fn search_should_not_return_deleted_documents_(deletion_strategy: DeletionStrategy) {
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
@ -961,7 +966,7 @@ mod tests {
.unwrap();
let deleted_internal_ids =
delete_documents(&mut wtxn, &index, &["1_7", "1_52"], disable_soft_deletion);
delete_documents(&mut wtxn, &index, &["1_7", "1_52"], deletion_strategy);
// search for abstract
let results = index.search(&wtxn).query("abstract").execute().unwrap();
@ -976,17 +981,17 @@ mod tests {
wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion);
db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
}
#[test]
fn search_should_not_return_deleted_documents() {
search_should_not_return_deleted_documents_(true);
search_should_not_return_deleted_documents_(false);
search_should_not_return_deleted_documents_(DeletionStrategy::AlwaysHard);
search_should_not_return_deleted_documents_(DeletionStrategy::AlwaysSoft);
}
fn geo_filtered_placeholder_search_should_not_return_deleted_documents_(
disable_soft_deletion: bool,
deletion_strategy: DeletionStrategy,
) {
let index = TempIndex::new();
@ -1024,7 +1029,7 @@ mod tests {
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
let deleted_internal_ids =
delete_documents(&mut wtxn, &index, &external_ids_to_delete, disable_soft_deletion);
delete_documents(&mut wtxn, &index, &external_ids_to_delete, deletion_strategy);
// Placeholder search with geo filter
let filter = Filter::from_str("_geoRadius(50.6924, 3.1763, 20000)").unwrap().unwrap();
@ -1040,18 +1045,22 @@ mod tests {
wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion);
db_snap!(index, facet_id_f64_docids, disable_soft_deletion);
db_snap!(index, facet_id_string_docids, disable_soft_deletion);
db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
db_snap!(index, facet_id_f64_docids, deletion_strategy);
db_snap!(index, facet_id_string_docids, deletion_strategy);
}
#[test]
fn geo_filtered_placeholder_search_should_not_return_deleted_documents() {
geo_filtered_placeholder_search_should_not_return_deleted_documents_(true);
geo_filtered_placeholder_search_should_not_return_deleted_documents_(false);
geo_filtered_placeholder_search_should_not_return_deleted_documents_(
DeletionStrategy::AlwaysHard,
);
geo_filtered_placeholder_search_should_not_return_deleted_documents_(
DeletionStrategy::AlwaysSoft,
);
}
fn get_documents_should_not_return_deleted_documents_(disable_soft_deletion: bool) {
fn get_documents_should_not_return_deleted_documents_(deletion_strategy: DeletionStrategy) {
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
@ -1094,7 +1103,7 @@ mod tests {
let deleted_external_ids = ["1_7", "1_52"];
let deleted_internal_ids =
delete_documents(&mut wtxn, &index, &deleted_external_ids, disable_soft_deletion);
delete_documents(&mut wtxn, &index, &deleted_external_ids, deletion_strategy);
// list all documents
let results = index.all_documents(&wtxn).unwrap();
@ -1125,16 +1134,16 @@ mod tests {
wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion);
db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
}
#[test]
fn get_documents_should_not_return_deleted_documents() {
get_documents_should_not_return_deleted_documents_(true);
get_documents_should_not_return_deleted_documents_(false);
get_documents_should_not_return_deleted_documents_(DeletionStrategy::AlwaysHard);
get_documents_should_not_return_deleted_documents_(DeletionStrategy::AlwaysSoft);
}
fn stats_should_not_return_deleted_documents_(disable_soft_deletion: bool) {
fn stats_should_not_return_deleted_documents_(deletion_strategy: DeletionStrategy) {
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
@ -1168,7 +1177,7 @@ mod tests {
{ "docid": "1_69", "label": ["geometry"]}
])).unwrap();
delete_documents(&mut wtxn, &index, &["1_7", "1_52"], disable_soft_deletion);
delete_documents(&mut wtxn, &index, &["1_7", "1_52"], deletion_strategy);
// count internal documents
let results = index.number_of_documents(&wtxn).unwrap();
@ -1182,12 +1191,12 @@ mod tests {
wtxn.commit().unwrap();
db_snap!(index, soft_deleted_documents_ids, disable_soft_deletion);
db_snap!(index, soft_deleted_documents_ids, deletion_strategy);
}
#[test]
fn stats_should_not_return_deleted_documents() {
stats_should_not_return_deleted_documents_(true);
stats_should_not_return_deleted_documents_(false);
stats_should_not_return_deleted_documents_(DeletionStrategy::AlwaysHard);
stats_should_not_return_deleted_documents_(DeletionStrategy::AlwaysSoft);
}
}

View File

@ -122,7 +122,7 @@ mod tests {
use crate::documents::documents_batch_reader_from_objects;
use crate::index::tests::TempIndex;
use crate::update::facet::test_helpers::ordered_string;
use crate::update::DeleteDocuments;
use crate::update::{DeleteDocuments, DeletionStrategy};
#[test]
fn delete_mixed_incremental_and_bulk() {
@ -165,7 +165,7 @@ mod tests {
let mut wtxn = index.env.write_txn().unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.disable_soft_deletion(true);
builder.strategy(DeletionStrategy::AlwaysHard);
builder.delete_documents(&RoaringBitmap::from_iter(0..100));
// by deleting the first 100 documents, we expect that:
// - the "id" part of the DB will be updated in bulk, since #affected_facet_value = 100 which is > database_len / 150 (= 13)
@ -224,7 +224,7 @@ mod tests {
let mut wtxn = index.env.write_txn().unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.disable_soft_deletion(true);
builder.strategy(DeletionStrategy::AlwaysHard);
builder.delete_documents(&RoaringBitmap::from_iter(0..100));
// by deleting the first 100 documents, we expect that:
// - the "id" part of the DB will be updated in bulk, since #affected_facet_value = 100 which is > database_len / 150 (= 13)
@ -283,7 +283,7 @@ mod tests {
for docid in docids_to_delete.into_iter().take(990) {
let mut wtxn = index.env.write_txn().unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.disable_soft_deletion(true);
builder.strategy(DeletionStrategy::AlwaysHard);
builder.delete_documents(&RoaringBitmap::from_iter([docid]));
builder.execute().unwrap();
wtxn.commit().unwrap();

View File

@ -463,11 +463,14 @@ mod tests {
use crate::db_snap;
use crate::documents::documents_batch_reader_from_objects;
use crate::index::tests::TempIndex;
use crate::update::DeletionStrategy;
#[test]
fn replace_all_identical_soft_deletion_then_hard_deletion() {
let mut index = TempIndex::new_with_map_size(4096 * 1000 * 100);
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index
.update_settings(|settings| {
settings.set_primary_key("id".to_owned());
@ -521,7 +524,7 @@ mod tests {
db_snap!(index, soft_deleted_documents_ids, "replaced_1_soft", @"6c975deb900f286d2f6456d2d5c3a123");
// Then replace the last document while disabling soft_deletion
index.index_documents_config.disable_soft_deletion = true;
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
let mut documents = vec![];
for i in 999..1000 {
documents.push(

View File

@ -163,7 +163,7 @@ mod tests {
use crate::db_snap;
use crate::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
use crate::index::tests::TempIndex;
use crate::update::{DeleteDocuments, IndexDocumentsMethod};
use crate::update::{DeleteDocuments, DeletionStrategy, IndexDocumentsMethod};
fn documents_with_enough_different_words_for_prefixes(
prefixes: &[&str],
@ -351,7 +351,7 @@ mod tests {
let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.disable_soft_deletion(true);
delete.strategy(DeletionStrategy::AlwaysHard);
delete.delete_documents(&RoaringBitmap::from_iter([50]));
delete.execute().unwrap();
wtxn.commit().unwrap();
@ -363,7 +363,7 @@ mod tests {
let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.disable_soft_deletion(true);
delete.strategy(DeletionStrategy::AlwaysHard);
delete.delete_documents(&RoaringBitmap::from_iter(0..50));
delete.execute().unwrap();
wtxn.commit().unwrap();
@ -435,6 +435,7 @@ mod tests {
let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.strategy(DeletionStrategy::AlwaysSoft);
delete.delete_documents(&RoaringBitmap::from_iter([50]));
delete.execute().unwrap();
wtxn.commit().unwrap();
@ -446,6 +447,8 @@ mod tests {
let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.strategy(DeletionStrategy::AlwaysSoft);
delete.delete_documents(&RoaringBitmap::from_iter(0..50));
delete.execute().unwrap();
wtxn.commit().unwrap();
@ -471,6 +474,7 @@ mod tests {
let mut index = TempIndex::new();
index.index_documents_config.words_prefix_threshold = Some(50);
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index
.update_settings(|settings| {
@ -530,7 +534,7 @@ mod tests {
fn replace_hard_deletion() {
let mut index = TempIndex::new();
index.index_documents_config.words_prefix_threshold = Some(50);
index.index_documents_config.disable_soft_deletion = true;
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index