snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 02:27:40 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Stop using removed delete pipelines almost everywhere

Browse Source
This commit is contained in:
Louis Dureuil 2023-10-25 13:40:46 +02:00
parent d651b3ef01
commit 2263dff02b
No known key found for this signature in database
6 changed files with 18 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
benchmarks/benches/indexing.rs
View File

@ -6,9 +6,7 @@ use std::path::Path;
use criterion::{criterion_group, criterion_main, Criterion}; use criterion::{criterion_group, criterion_main, Criterion};
use milli::heed::{EnvOpenOptions, RwTxn}; use milli::heed::{EnvOpenOptions, RwTxn};
use milli::update::{ use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
DeleteDocuments, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings,
};
use milli::Index; use milli::Index;
use rand::seq::SliceRandom; use rand::seq::SliceRandom;
use rand_chacha::rand_core::SeedableRng; use rand_chacha::rand_core::SeedableRng;

32
milli/src/index.rs
View File

@ -1469,8 +1469,7 @@ pub(crate) mod tests {
use crate::error::{Error, InternalError}; use crate::error::{Error, InternalError};
use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS}; use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS};
use crate::update::{ use crate::update::{
self, DeleteDocuments, DeletionStrategy, IndexDocuments, IndexDocumentsConfig, self, IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Settings,
IndexDocumentsMethod, IndexerConfig, Settings,
}; };
use crate::{db_snap, obkv_to_json, Filter, Index, Search, SearchResult}; use crate::{db_snap, obkv_to_json, Filter, Index, Search, SearchResult};
@ -1563,11 +1562,20 @@ pub(crate) mod tests {
pub fn delete_document(&self, external_document_id: &str) { pub fn delete_document(&self, external_document_id: &str) {
let mut wtxn = self.write_txn().unwrap(); let mut wtxn = self.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, self).unwrap(); let builder = IndexDocuments::new(
delete.strategy(self.index_documents_config.deletion_strategy); &mut wtxn,
self,
&self.indexer_config,
self.index_documents_config.clone(),
|_| (),
|| false,
)
.unwrap();
let (builder, user_error) =
builder.remove_documents(vec![external_document_id.to_owned()]).unwrap();
user_error.unwrap();
builder.execute().unwrap();
delete.delete_external_id(external_document_id);
delete.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
} }
} }
@ -1884,7 +1892,6 @@ pub(crate) mod tests {
use maplit::hashset; use maplit::hashset;
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
let index = index; let index = index;
index index
@ -2055,8 +2062,6 @@ pub(crate) mod tests {
} }
// Second Batch: replace the documents with soft-deletion // Second Batch: replace the documents with soft-deletion
{ {
index.index_documents_config.deletion_strategy =
crate::update::DeletionStrategy::AlwaysSoft;
let mut docs1 = vec![]; let mut docs1 = vec![];
for i in 0..3 { for i in 0..3 {
docs1.push(serde_json::json!( docs1.push(serde_json::json!(
@ -2125,7 +2130,6 @@ pub(crate) mod tests {
drop(rtxn); drop(rtxn);
// Third Batch: replace the documents with soft-deletion again // Third Batch: replace the documents with soft-deletion again
{ {
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
let mut docs1 = vec![]; let mut docs1 = vec![];
for i in 0..3 { for i in 0..3 {
docs1.push(serde_json::json!( docs1.push(serde_json::json!(
@ -2194,7 +2198,6 @@ pub(crate) mod tests {
// Fourth Batch: replace the documents without soft-deletion // Fourth Batch: replace the documents without soft-deletion
{ {
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
let mut docs1 = vec![]; let mut docs1 = vec![];
for i in 0..3 { for i in 0..3 {
docs1.push(serde_json::json!( docs1.push(serde_json::json!(
@ -2266,7 +2269,6 @@ pub(crate) mod tests {
fn bug_3021_first() { fn bug_3021_first() {
// https://github.com/meilisearch/meilisearch/issues/3021 // https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index index
@ -2379,7 +2381,6 @@ pub(crate) mod tests {
fn bug_3021_second() { fn bug_3021_second() {
// https://github.com/meilisearch/meilisearch/issues/3021 // https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
index index
@ -2505,7 +2506,6 @@ pub(crate) mod tests {
fn bug_3021_third() { fn bug_3021_third() {
// https://github.com/meilisearch/meilisearch/issues/3021 // https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
index index
@ -2544,8 +2544,6 @@ pub(crate) mod tests {
"###); "###);
db_snap!(index, soft_deleted_documents_ids, 2, @"[0, ]"); db_snap!(index, soft_deleted_documents_ids, 2, @"[0, ]");
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
index.add_documents(documents!([{ "primary_key": "4", "a": 2 }])).unwrap(); index.add_documents(documents!([{ "primary_key": "4", "a": 2 }])).unwrap();
db_snap!(index, documents_ids, @"[2, 3, ]"); db_snap!(index, documents_ids, @"[2, 3, ]");
@ -2579,7 +2577,6 @@ pub(crate) mod tests {
// https://github.com/meilisearch/meilisearch/issues/3021 // https://github.com/meilisearch/meilisearch/issues/3021
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index index
.update_settings(|settings| { .update_settings(|settings| {
@ -2622,7 +2619,6 @@ pub(crate) mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.strategy(DeletionStrategy::AlwaysHard);
delete.execute().unwrap(); delete.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

4
milli/src/update/facet/mod.rs
View File

@ -563,14 +563,11 @@ mod tests {
use crate::db_snap; use crate::db_snap;
use crate::documents::documents_batch_reader_from_objects; use crate::documents::documents_batch_reader_from_objects;
use crate::index::tests::TempIndex; use crate::index::tests::TempIndex;
use crate::update::DeletionStrategy;
#[test] #[test]
fn replace_all_identical_soft_deletion_then_hard_deletion() { fn replace_all_identical_soft_deletion_then_hard_deletion() {
let mut index = TempIndex::new_with_map_size(4096 * 1000 * 100); let mut index = TempIndex::new_with_map_size(4096 * 1000 * 100);
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_primary_key("id".to_owned()); settings.set_primary_key("id".to_owned());
@ -622,7 +619,6 @@ mod tests {
db_snap!(index, soft_deleted_documents_ids, "replaced_1_soft", @"6c975deb900f286d2f6456d2d5c3a123"); db_snap!(index, soft_deleted_documents_ids, "replaced_1_soft", @"6c975deb900f286d2f6456d2d5c3a123");
// Then replace the last document while disabling soft_deletion // Then replace the last document while disabling soft_deletion
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
let mut documents = vec![]; let mut documents = vec![];
for i in 999..1000 { for i in 999..1000 {
documents.push( documents.push(

7
milli/src/update/index_documents/mod.rs
View File

@ -35,8 +35,8 @@ use crate::documents::{obkv_to_object, DocumentsBatchReader};
use crate::error::{Error, InternalError, UserError}; use crate::error::{Error, InternalError, UserError};
pub use crate::update::index_documents::helpers::CursorClonableMmap; pub use crate::update::index_documents::helpers::CursorClonableMmap;
use crate::update::{ use crate::update::{
DeletionStrategy, IndexerConfig, PrefixWordPairsProximityDocids, UpdateIndexingStep, IndexerConfig, PrefixWordPairsProximityDocids, UpdateIndexingStep, WordPrefixDocids,
WordPrefixDocids, WordPrefixIntegerDocids, WordsPrefixesFst, WordPrefixIntegerDocids, WordsPrefixesFst,
}; };
use crate::{CboRoaringBitmapCodec, Index, Result}; use crate::{CboRoaringBitmapCodec, Index, Result};
@ -89,7 +89,6 @@ pub struct IndexDocumentsConfig {
pub words_positions_level_group_size: Option<NonZeroU32>, pub words_positions_level_group_size: Option<NonZeroU32>,
pub words_positions_min_level_size: Option<NonZeroU32>, pub words_positions_min_level_size: Option<NonZeroU32>,
pub update_method: IndexDocumentsMethod, pub update_method: IndexDocumentsMethod,
pub deletion_strategy: DeletionStrategy,
pub autogenerate_docids: bool, pub autogenerate_docids: bool,
} }
@ -2497,7 +2496,6 @@ mod tests {
// Delete not all of the documents but some of them. // Delete not all of the documents but some of them.
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.strategy(DeletionStrategy::AlwaysHard);
builder.delete_external_id("0"); builder.delete_external_id("0");
builder.delete_external_id("3"); builder.delete_external_id("3");
let result = builder.execute().unwrap(); let result = builder.execute().unwrap();
@ -2559,7 +2557,6 @@ mod tests {
] ]
*/ */
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
// START OF BATCH // START OF BATCH

1
milli/src/update/mod.rs
View File

@ -1,6 +1,5 @@
pub use self::available_documents_ids::AvailableDocumentsIds; pub use self::available_documents_ids::AvailableDocumentsIds;
pub use self::clear_documents::ClearDocuments; pub use self::clear_documents::ClearDocuments;
pub use self::delete_documents::{DeleteDocuments, DeletionStrategy, DocumentDeletionResult};
pub use self::facet::bulk::FacetsUpdateBulk; pub use self::facet::bulk::FacetsUpdateBulk;
pub use self::facet::incremental::FacetsUpdateIncrementalInner; pub use self::facet::incremental::FacetsUpdateIncrementalInner;
pub use self::index_documents::{ pub use self::index_documents::{

8
milli/src/update/prefix_word_pairs/mod.rs
View File

@ -149,7 +149,7 @@ mod tests {
use crate::db_snap; use crate::db_snap;
use crate::documents::{DocumentsBatchBuilder, DocumentsBatchReader}; use crate::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
use crate::index::tests::TempIndex; use crate::index::tests::TempIndex;
use crate::update::{DeleteDocuments, DeletionStrategy, IndexDocumentsMethod}; use crate::update::IndexDocumentsMethod;
fn documents_with_enough_different_words_for_prefixes( fn documents_with_enough_different_words_for_prefixes(
prefixes: &[&str], prefixes: &[&str],
@ -337,7 +337,6 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.strategy(DeletionStrategy::AlwaysHard);
delete.delete_documents(&RoaringBitmap::from_iter([50])); delete.delete_documents(&RoaringBitmap::from_iter([50]));
delete.execute().unwrap(); delete.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@ -349,7 +348,6 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.strategy(DeletionStrategy::AlwaysHard);
delete.delete_documents(&RoaringBitmap::from_iter(0..50)); delete.delete_documents(&RoaringBitmap::from_iter(0..50));
delete.execute().unwrap(); delete.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@ -421,7 +419,6 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.strategy(DeletionStrategy::AlwaysSoft);
delete.delete_documents(&RoaringBitmap::from_iter([50])); delete.delete_documents(&RoaringBitmap::from_iter([50]));
delete.execute().unwrap(); delete.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@ -433,7 +430,6 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut delete = DeleteDocuments::new(&mut wtxn, &index).unwrap();
delete.strategy(DeletionStrategy::AlwaysSoft);
delete.delete_documents(&RoaringBitmap::from_iter(0..50)); delete.delete_documents(&RoaringBitmap::from_iter(0..50));
delete.execute().unwrap(); delete.execute().unwrap();
@ -460,7 +456,6 @@ mod tests {
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.words_prefix_threshold = Some(50); index.index_documents_config.words_prefix_threshold = Some(50);
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysSoft;
index index
.update_settings(|settings| { .update_settings(|settings| {
@ -520,7 +515,6 @@ mod tests {
fn replace_hard_deletion() { fn replace_hard_deletion() {
let mut index = TempIndex::new(); let mut index = TempIndex::new();
index.index_documents_config.words_prefix_threshold = Some(50); index.index_documents_config.words_prefix_threshold = Some(50);
index.index_documents_config.deletion_strategy = DeletionStrategy::AlwaysHard;
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index index