snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 02:27:40 +08:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Correctly batch tasks with different index creation rights

Browse Source
This commit is contained in:
Clément Renault 2022-10-06 15:55:48 +02:00
parent 87212cfd20
commit 068a4b2884
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
4 changed files with 255 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

320
index-scheduler/src/autobatcher.rs
View File

@ -10,6 +10,7 @@ pub enum BatchKind {
}, },
DocumentImport { DocumentImport {
method: IndexDocumentsMethod, method: IndexDocumentsMethod,
allow_index_creation: bool,
import_ids: Vec<TaskId>, import_ids: Vec<TaskId>,
}, },
DocumentDeletion { DocumentDeletion {
@ -17,14 +18,17 @@ pub enum BatchKind {
}, },
ClearAndSettings { ClearAndSettings {
other: Vec<TaskId>, other: Vec<TaskId>,
allow_index_creation: bool,
settings_ids: Vec<TaskId>, settings_ids: Vec<TaskId>,
}, },
SettingsAndDocumentImport { SettingsAndDocumentImport {
settings_ids: Vec<TaskId>, settings_ids: Vec<TaskId>,
method: IndexDocumentsMethod, method: IndexDocumentsMethod,
allow_index_creation: bool,
import_ids: Vec<TaskId>, import_ids: Vec<TaskId>,
}, },
Settings { Settings {
allow_index_creation: bool,
settings_ids: Vec<TaskId>, settings_ids: Vec<TaskId>,
}, },
IndexDeletion { IndexDeletion {
@ -50,18 +54,21 @@ impl BatchKind {
Kind::IndexUpdate => Break(BatchKind::IndexUpdate { id: task_id }), Kind::IndexUpdate => Break(BatchKind::IndexUpdate { id: task_id }),
Kind::IndexSwap => Break(BatchKind::IndexSwap { id: task_id }), Kind::IndexSwap => Break(BatchKind::IndexSwap { id: task_id }),
Kind::DocumentClear => Continue(BatchKind::DocumentClear { ids: vec![task_id] }), Kind::DocumentClear => Continue(BatchKind::DocumentClear { ids: vec![task_id] }),
Kind::DocumentAddition => Continue(BatchKind::DocumentImport { Kind::DocumentImport {
method: ReplaceDocuments, method,
import_ids: vec![task_id], allow_index_creation,
}), } => Continue(BatchKind::DocumentImport {
Kind::DocumentUpdate => Continue(BatchKind::DocumentImport { method,
method: UpdateDocuments, allow_index_creation,
import_ids: vec![task_id], import_ids: vec![task_id],
}), }),
Kind::DocumentDeletion => Continue(BatchKind::DocumentDeletion { Kind::DocumentDeletion => Continue(BatchKind::DocumentDeletion {
deletion_ids: vec![task_id], deletion_ids: vec![task_id],
}), }),
Kind::Settings => Continue(BatchKind::Settings { Kind::Settings {
allow_index_creation,
} => Continue(BatchKind::Settings {
allow_index_creation,
settings_ids: vec![task_id], settings_ids: vec![task_id],
}), }),
Kind::DumpExport | Kind::Snapshot | Kind::CancelTask => unreachable!(), Kind::DumpExport | Kind::Snapshot | Kind::CancelTask => unreachable!(),
@ -69,6 +76,7 @@ impl BatchKind {
} }
/// Returns a `ControlFlow::Break` if you must stop right now. /// Returns a `ControlFlow::Break` if you must stop right now.
#[rustfmt::skip]
fn accumulate(self, id: TaskId, kind: Kind) -> ControlFlow<BatchKind, BatchKind> { fn accumulate(self, id: TaskId, kind: Kind) -> ControlFlow<BatchKind, BatchKind> {
match (self, kind) { match (self, kind) {
// We don't batch any of these operations // We don't batch any of these operations
@ -76,31 +84,17 @@ impl BatchKind {
// The index deletion can batch with everything but must stop after // The index deletion can batch with everything but must stop after
( (
BatchKind::DocumentClear { mut ids } BatchKind::DocumentClear { mut ids }
| BatchKind::DocumentImport { | BatchKind::DocumentDeletion { deletion_ids: mut ids }
method: _, | BatchKind::DocumentImport { method: _, allow_index_creation: _, import_ids: mut ids }
import_ids: mut ids, | BatchKind::Settings { allow_index_creation: _, settings_ids: mut ids },
}
| BatchKind::DocumentDeletion {
deletion_ids: mut ids,
}
| BatchKind::Settings {
settings_ids: mut ids,
},
Kind::IndexDeletion, Kind::IndexDeletion,
) => { ) => {
ids.push(id); ids.push(id);
Break(BatchKind::IndexDeletion { ids }) Break(BatchKind::IndexDeletion { ids })
} }
( (
BatchKind::ClearAndSettings { BatchKind::ClearAndSettings { settings_ids: mut ids, allow_index_creation: _, mut other }
settings_ids: mut ids, | BatchKind::SettingsAndDocumentImport { import_ids: mut ids, method: _, allow_index_creation: _, settings_ids: mut other },
mut other,
}
| BatchKind::SettingsAndDocumentImport {
import_ids: mut ids,
method: _,
settings_ids: mut other,
},
Kind::IndexDeletion, Kind::IndexDeletion,
) => { ) => {
ids.push(id); ids.push(id);
@ -117,110 +111,125 @@ impl BatchKind {
} }
( (
this @ BatchKind::DocumentClear { .. }, this @ BatchKind::DocumentClear { .. },
Kind::DocumentAddition | Kind::DocumentUpdate | Kind::Settings, Kind::DocumentImport { .. } | Kind::Settings { .. },
) => Break(this), ) => Break(this),
( (
BatchKind::DocumentImport { BatchKind::DocumentImport { method: _, allow_index_creation: _, import_ids: mut ids },
method: _,
import_ids: mut ids,
},
Kind::DocumentClear, Kind::DocumentClear,
) => { ) => {
ids.push(id); ids.push(id);
Continue(BatchKind::DocumentClear { ids }) Continue(BatchKind::DocumentClear { ids })
} }
// We only want to batch together document imports that are allowed to create the index
// or document imports not allowed to create an index if the first operation can.
(
this @ BatchKind::DocumentImport { method: _, allow_index_creation: false, .. },
Kind::DocumentImport { method: _, allow_index_creation: true },
) => Break(this),
// we can autobatch the same kind of document additions / updates // we can autobatch the same kind of document additions / updates
( (
BatchKind::DocumentImport { BatchKind::DocumentImport { method: ReplaceDocuments, allow_index_creation, mut import_ids },
method: ReplaceDocuments, Kind::DocumentImport { method: ReplaceDocuments, .. },
mut import_ids,
},
Kind::DocumentAddition,
) => { ) => {
import_ids.push(id); import_ids.push(id);
Continue(BatchKind::DocumentImport { Continue(BatchKind::DocumentImport {
method: ReplaceDocuments, method: ReplaceDocuments,
allow_index_creation,
import_ids, import_ids,
}) })
} }
( (
BatchKind::DocumentImport { BatchKind::DocumentImport { method: UpdateDocuments, allow_index_creation, mut import_ids },
method: UpdateDocuments, Kind::DocumentImport { method: UpdateDocuments, .. },
mut import_ids,
},
Kind::DocumentUpdate,
) => { ) => {
import_ids.push(id); import_ids.push(id);
Continue(BatchKind::DocumentImport { Continue(BatchKind::DocumentImport {
method: UpdateDocuments, method: UpdateDocuments,
allow_index_creation,
import_ids, import_ids,
}) })
} }
// but we can't autobatch documents if it's not the same kind // but we can't autobatch documents if it's not the same kind
// this match branch MUST be AFTER the previous one // this match branch MUST be AFTER the previous one
( (
this @ BatchKind::DocumentImport { .. }, this @ BatchKind::DocumentImport { .. },
Kind::DocumentDeletion | Kind::DocumentAddition | Kind::DocumentUpdate, Kind::DocumentDeletion | Kind::DocumentImport { .. },
) => Break(this), ) => Break(this),
(BatchKind::DocumentImport { method, import_ids }, Kind::Settings) => {
Continue(BatchKind::SettingsAndDocumentImport { // We only want to batch together document imports that are allowed to create the index
// or document imports not allowed to create an index if the first operation can.
(
this @ BatchKind::DocumentImport { allow_index_creation: false, .. },
Kind::Settings { allow_index_creation: true },
) => Break(this),
(
BatchKind::DocumentImport { method, allow_index_creation, import_ids },
Kind::Settings { .. },
) => Continue(BatchKind::SettingsAndDocumentImport {
settings_ids: vec![id], settings_ids: vec![id],
method, method,
allow_index_creation,
import_ids, import_ids,
}) }),
}
(BatchKind::DocumentDeletion { mut deletion_ids }, Kind::DocumentClear) => { (BatchKind::DocumentDeletion { mut deletion_ids }, Kind::DocumentClear) => {
deletion_ids.push(id); deletion_ids.push(id);
Continue(BatchKind::DocumentClear { ids: deletion_ids }) Continue(BatchKind::DocumentClear { ids: deletion_ids })
} }
( (this @ BatchKind::DocumentDeletion { .. }, Kind::DocumentImport { .. }) => Break(this),
this @ BatchKind::DocumentDeletion { .. },
Kind::DocumentAddition | Kind::DocumentUpdate,
) => Break(this),
(BatchKind::DocumentDeletion { mut deletion_ids }, Kind::DocumentDeletion) => { (BatchKind::DocumentDeletion { mut deletion_ids }, Kind::DocumentDeletion) => {
deletion_ids.push(id); deletion_ids.push(id);
Continue(BatchKind::DocumentDeletion { deletion_ids }) Continue(BatchKind::DocumentDeletion { deletion_ids })
} }
(this @ BatchKind::DocumentDeletion { .. }, Kind::Settings) => Break(this), (this @ BatchKind::DocumentDeletion { .. }, Kind::Settings { .. }) => Break(this),
(BatchKind::Settings { settings_ids }, Kind::DocumentClear) => { (
Continue(BatchKind::ClearAndSettings { BatchKind::Settings { settings_ids, allow_index_creation },
Kind::DocumentClear,
) => Continue(BatchKind::ClearAndSettings {
settings_ids: settings_ids, settings_ids: settings_ids,
allow_index_creation,
other: vec![id], other: vec![id],
}) }),
}
( (
this @ BatchKind::Settings { .. }, this @ BatchKind::Settings { .. },
Kind::DocumentAddition | Kind::DocumentUpdate | Kind::DocumentDeletion, Kind::DocumentImport { .. } | Kind::DocumentDeletion,
) => Break(this), ) => Break(this),
(BatchKind::Settings { mut settings_ids }, Kind::Settings) => { (
this @ BatchKind::Settings { allow_index_creation: false, .. },
Kind::Settings { allow_index_creation: true },
) => Break(this),
(
BatchKind::Settings { mut settings_ids, allow_index_creation },
Kind::Settings { .. },
) => {
settings_ids.push(id); settings_ids.push(id);
Continue(BatchKind::Settings { settings_ids }) Continue(BatchKind::Settings {
allow_index_creation,
settings_ids,
})
} }
( (
BatchKind::ClearAndSettings { BatchKind::ClearAndSettings { mut other, settings_ids, allow_index_creation },
mut other,
settings_ids,
},
Kind::DocumentClear, Kind::DocumentClear,
) => { ) => {
other.push(id); other.push(id);
Continue(BatchKind::ClearAndSettings { Continue(BatchKind::ClearAndSettings {
other, other,
settings_ids, settings_ids,
allow_index_creation,
}) })
} }
( (this @ BatchKind::ClearAndSettings { .. }, Kind::DocumentImport { .. }) => Break(this),
this @ BatchKind::ClearAndSettings { .. },
Kind::DocumentAddition | Kind::DocumentUpdate,
) => Break(this),
( (
BatchKind::ClearAndSettings { BatchKind::ClearAndSettings {
mut other, mut other,
settings_ids, settings_ids,
allow_index_creation,
}, },
Kind::DocumentDeletion, Kind::DocumentDeletion,
) => { ) => {
@ -228,64 +237,64 @@ impl BatchKind {
Continue(BatchKind::ClearAndSettings { Continue(BatchKind::ClearAndSettings {
other, other,
settings_ids, settings_ids,
allow_index_creation,
}) })
} }
( (
BatchKind::ClearAndSettings { this @ BatchKind::ClearAndSettings { allow_index_creation: false, .. },
mut settings_ids, Kind::Settings {
other, allow_index_creation: true,
}, },
Kind::Settings, ) => Break(this),
(
BatchKind::ClearAndSettings { mut settings_ids, other, allow_index_creation },
Kind::Settings { .. },
) => { ) => {
settings_ids.push(id); settings_ids.push(id);
Continue(BatchKind::ClearAndSettings { Continue(BatchKind::ClearAndSettings {
other, other,
settings_ids, settings_ids,
allow_index_creation,
}) })
} }
( (
BatchKind::SettingsAndDocumentImport { BatchKind::SettingsAndDocumentImport { settings_ids, method: _, import_ids: mut other, allow_index_creation },
settings_ids,
method: _,
import_ids: mut other,
},
Kind::DocumentClear, Kind::DocumentClear,
) => { ) => {
other.push(id); other.push(id);
Continue(BatchKind::ClearAndSettings { Continue(BatchKind::ClearAndSettings {
settings_ids, settings_ids,
other, other,
allow_index_creation,
}) })
} }
// we can batch the settings with a kind of document operation with the same kind of document operation // we can batch the settings with a kind of document operation with the same kind of document operation
( (
BatchKind::SettingsAndDocumentImport { this @ BatchKind::SettingsAndDocumentImport { allow_index_creation: false, .. },
settings_ids, Kind::DocumentImport { allow_index_creation: true, .. },
method: ReplaceDocuments, ) => Break(this),
mut import_ids, (
}, BatchKind::SettingsAndDocumentImport { settings_ids, method: ReplaceDocuments, mut import_ids, allow_index_creation },
Kind::DocumentAddition, Kind::DocumentImport { method: ReplaceDocuments, .. },
) => { ) => {
import_ids.push(id); import_ids.push(id);
Continue(BatchKind::SettingsAndDocumentImport { Continue(BatchKind::SettingsAndDocumentImport {
settings_ids, settings_ids,
method: ReplaceDocuments, method: ReplaceDocuments,
allow_index_creation,
import_ids, import_ids,
}) })
} }
( (
BatchKind::SettingsAndDocumentImport { BatchKind::SettingsAndDocumentImport { settings_ids, method: UpdateDocuments, allow_index_creation, mut import_ids },
settings_ids, Kind::DocumentImport { method: UpdateDocuments, .. },
method: UpdateDocuments,
mut import_ids,
},
Kind::DocumentUpdate,
) => { ) => {
import_ids.push(id); import_ids.push(id);
Continue(BatchKind::SettingsAndDocumentImport { Continue(BatchKind::SettingsAndDocumentImport {
settings_ids, settings_ids,
method: UpdateDocuments, method: UpdateDocuments,
allow_index_creation,
import_ids, import_ids,
}) })
} }
@ -293,20 +302,21 @@ impl BatchKind {
// this MUST be AFTER the two previous branch // this MUST be AFTER the two previous branch
( (
this @ BatchKind::SettingsAndDocumentImport { .. }, this @ BatchKind::SettingsAndDocumentImport { .. },
Kind::DocumentDeletion | Kind::DocumentAddition | Kind::DocumentUpdate, Kind::DocumentDeletion | Kind::DocumentImport { .. },
) => Break(this), ) => Break(this),
( (
BatchKind::SettingsAndDocumentImport { this @ BatchKind::SettingsAndDocumentImport { allow_index_creation: false, .. },
mut settings_ids, Kind::Settings { allow_index_creation: true },
method, ) => Break(this),
import_ids, (
}, BatchKind::SettingsAndDocumentImport { mut settings_ids, method, allow_index_creation, import_ids },
Kind::Settings, Kind::Settings { .. },
) => { ) => {
settings_ids.push(id); settings_ids.push(id);
Continue(BatchKind::SettingsAndDocumentImport { Continue(BatchKind::SettingsAndDocumentImport {
settings_ids, settings_ids,
method, method,
allow_index_creation,
import_ids, import_ids,
}) })
} }
@ -362,119 +372,129 @@ mod tests {
#[test] #[test]
fn autobatch_simple_operation_together() { fn autobatch_simple_operation_together() {
// we can autobatch one or multiple DocumentAddition together // we can autobatch one or multiple DocumentAddition together
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentAddition]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0, 1, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
// we can autobatch one or multiple DocumentUpdate together // we can autobatch one or multiple DocumentUpdate together
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentUpdate]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0, 1, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })");
// we can autobatch one or multiple DocumentDeletion together // we can autobatch one or multiple DocumentDeletion together
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion]), @"Some(DocumentDeletion { deletion_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentDeletion, DocumentDeletion]), @"Some(DocumentDeletion { deletion_ids: [0, 1, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentDeletion, DocumentDeletion]), @"Some(DocumentDeletion { deletion_ids: [0, 1, 2] })");
// we can autobatch one or multiple Settings together // we can autobatch one or multiple Settings together
assert_smol_debug_snapshot!(autobatch_from([Settings]), @"Some(Settings { settings_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }]), @"Some(Settings { allow_index_creation: true, settings_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([Settings, Settings, Settings]), @"Some(Settings { settings_ids: [0, 1, 2] })"); assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }, Settings { allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(Settings { allow_index_creation: true, settings_ids: [0, 1, 2] })");
} }
#[test] #[test]
fn simple_document_operation_dont_autobatch_with_other() { fn simple_document_operation_dont_autobatch_with_other() {
// addition, updates and deletion can't batch together // addition, updates and deletion can't batch together
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentUpdate]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentDeletion]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentDeletion]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentAddition]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentDeletion]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentDeletion]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentAddition]), @"Some(DocumentDeletion { deletion_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentUpdate]), @"Some(DocumentDeletion { deletion_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexCreation]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexCreation]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexCreation]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexCreation]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexCreation]), @"Some(DocumentDeletion { deletion_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexCreation]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexUpdate]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexUpdate]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexUpdate]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexUpdate]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexUpdate]), @"Some(DocumentDeletion { deletion_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexUpdate]), @"Some(DocumentDeletion { deletion_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexSwap]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexSwap]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexSwap]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexSwap]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexSwap]), @"Some(DocumentDeletion { deletion_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexSwap]), @"Some(DocumentDeletion { deletion_ids: [0] })");
} }
#[test] #[test]
fn document_addition_batch_with_settings() { fn document_addition_batch_with_settings() {
// simple case // simple case
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
// multiple settings and doc addition // multiple settings and doc addition
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, Settings, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, import_ids: [0, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, Settings, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, import_ids: [0, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
// addition and setting unordered // addition and setting unordered
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentAddition, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: ReplaceDocuments, import_ids: [0, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 2] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentUpdate, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: UpdateDocuments, import_ids: [0, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 2] })");
// We ensure this kind of batch doesn't batch with forbidden operations // We ensure this kind of batch doesn't batch with forbidden operations
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentAddition]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })");
} }
#[test] #[test]
fn clear_and_additions() { fn clear_and_additions() {
// these two doesn't need to batch // these two doesn't need to batch
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentAddition]), @"Some(DocumentClear { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentUpdate]), @"Some(DocumentClear { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0] })");
// Basic use case // Basic use case
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })");
// This batch kind doesn't mix with other document addition // This batch kind doesn't mix with other document addition
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentAddition]), @"Some(DocumentClear { ids: [0, 1, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentClear, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0, 1, 2] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentUpdate]), @"Some(DocumentClear { ids: [0, 1, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentClear, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0, 1, 2] })");
// But you can batch multiple clear together // But you can batch multiple clear together
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })");
} }
#[test] #[test]
fn clear_and_additions_and_settings() { fn clear_and_additions_and_settings() {
// A clear don't need to autobatch the settings that happens AFTER there is no documents // A clear don't need to autobatch the settings that happens AFTER there is no documents
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, Settings]), @"Some(DocumentClear { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentClear, Settings { allow_index_creation: true }]), @"Some(DocumentClear { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([Settings, DocumentClear, Settings]), @"Some(ClearAndSettings { other: [1], settings_ids: [0, 2] })"); assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }, DocumentClear, Settings { allow_index_creation: true }]), @"Some(ClearAndSettings { other: [1], allow_index_creation: true, settings_ids: [0, 2] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], settings_ids: [1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], settings_ids: [1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })");
} }
#[test] #[test]
fn anything_and_index_deletion() { fn anything_and_index_deletion() {
// The indexdeletion doesn't batch with anything that happens AFTER // The indexdeletion doesn't batch with anything that happens AFTER
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentAddition]), @"Some(IndexDeletion { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(IndexDeletion { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentUpdate]), @"Some(IndexDeletion { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(IndexDeletion { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentDeletion]), @"Some(IndexDeletion { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentDeletion]), @"Some(IndexDeletion { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentClear]), @"Some(IndexDeletion { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentClear]), @"Some(IndexDeletion { ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, Settings]), @"Some(IndexDeletion { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, Settings { allow_index_creation: true }]), @"Some(IndexDeletion { ids: [0] })");
// The index deletion can accept almost any type of BatchKind and transform it to an IndexDeletion // The index deletion can accept almost any type of BatchKind and transform it to an IndexDeletion
// First, the basic cases // First, the basic cases
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([Settings, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })");
// Then the mixed cases // Then the mixed cases
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })");
}
#[test]
fn allowed_and_disallowed_index_creation() {
// DocumentImport that can create indexes can't be mixed with those disallowed to do so
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: false }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: false }, DocumentImport { method: ReplaceDocuments, allow_index_creation: false }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0, 1] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })");
assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: false }, Settings { allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })");
} }
} }

51
index-scheduler/src/batch.rs
View File

@ -4,7 +4,7 @@ use crate::{
Error, IndexScheduler, Result, TaskId, Error, IndexScheduler, Result, TaskId,
}; };
use index::apply_settings_to_builder; use index::apply_settings_to_builder;
use index::error::{IndexError, MilliError}; use index::error::IndexError;
use index::{Settings, Unchecked}; use index::{Settings, Unchecked};
use log::{debug, info}; use log::{debug, info};
use milli::documents::DocumentsBatchReader; use milli::documents::DocumentsBatchReader;
@ -39,6 +39,7 @@ pub(crate) enum IndexOperation {
index_uid: String, index_uid: String,
primary_key: Option<String>, primary_key: Option<String>,
method: IndexDocumentsMethod, method: IndexDocumentsMethod,
allow_index_creation: bool,
documents_counts: Vec<u64>, documents_counts: Vec<u64>,
content_files: Vec<Uuid>, content_files: Vec<Uuid>,
tasks: Vec<Task>, tasks: Vec<Task>,
@ -56,6 +57,7 @@ pub(crate) enum IndexOperation {
index_uid: String, index_uid: String,
// TODO what's that boolean, does it mean that it removes things or what? // TODO what's that boolean, does it mean that it removes things or what?
settings: Vec<(bool, Settings<Unchecked>)>, settings: Vec<(bool, Settings<Unchecked>)>,
allow_index_creation: bool,
tasks: Vec<Task>, tasks: Vec<Task>,
}, },
DocumentClearAndSetting { DocumentClearAndSetting {
@ -64,6 +66,7 @@ pub(crate) enum IndexOperation {
// TODO what's that boolean, does it mean that it removes things or what? // TODO what's that boolean, does it mean that it removes things or what?
settings: Vec<(bool, Settings<Unchecked>)>, settings: Vec<(bool, Settings<Unchecked>)>,
allow_index_creation: bool,
settings_tasks: Vec<Task>, settings_tasks: Vec<Task>,
}, },
SettingsAndDocumentImport { SettingsAndDocumentImport {
@ -71,6 +74,7 @@ pub(crate) enum IndexOperation {
primary_key: Option<String>, primary_key: Option<String>,
method: IndexDocumentsMethod, method: IndexDocumentsMethod,
allow_index_creation: bool,
documents_counts: Vec<u64>, documents_counts: Vec<u64>,
content_files: Vec<Uuid>, content_files: Vec<Uuid>,
document_import_tasks: Vec<Task>, document_import_tasks: Vec<Task>,
@ -126,7 +130,11 @@ impl IndexScheduler {
index_uid, index_uid,
}))) })))
} }
BatchKind::DocumentImport { method, import_ids } => { BatchKind::DocumentImport {
method,
import_ids,
allow_index_creation,
} => {
let tasks = self.get_existing_tasks(rtxn, import_ids)?; let tasks = self.get_existing_tasks(rtxn, import_ids)?;
let primary_key = match &tasks[0].kind { let primary_key = match &tasks[0].kind {
KindWithContent::DocumentImport { primary_key, .. } => primary_key.clone(), KindWithContent::DocumentImport { primary_key, .. } => primary_key.clone(),
@ -154,6 +162,7 @@ impl IndexScheduler {
index_uid, index_uid,
primary_key, primary_key,
method, method,
allow_index_creation,
documents_counts, documents_counts,
content_files, content_files,
tasks, tasks,
@ -181,7 +190,10 @@ impl IndexScheduler {
}, },
))) )))
} }
BatchKind::Settings { settings_ids } => { BatchKind::Settings {
settings_ids,
allow_index_creation,
} => {
let tasks = self.get_existing_tasks(rtxn, settings_ids)?; let tasks = self.get_existing_tasks(rtxn, settings_ids)?;
let mut settings = Vec::new(); let mut settings = Vec::new();
@ -199,20 +211,30 @@ impl IndexScheduler {
Ok(Some(Batch::IndexOperation(IndexOperation::Settings { Ok(Some(Batch::IndexOperation(IndexOperation::Settings {
index_uid, index_uid,
settings, settings,
allow_index_creation,
tasks, tasks,
}))) })))
} }
BatchKind::ClearAndSettings { BatchKind::ClearAndSettings {
other, other,
settings_ids, settings_ids,
allow_index_creation,
} => { } => {
let (index_uid, settings, settings_tasks) = match self let (index_uid, settings, settings_tasks) = match self
.create_next_batch_index(rtxn, index_uid, BatchKind::Settings { settings_ids })? .create_next_batch_index(
rtxn,
index_uid,
BatchKind::Settings {
settings_ids,
allow_index_creation,
},
)?
.unwrap() .unwrap()
{ {
Batch::IndexOperation(IndexOperation::Settings { Batch::IndexOperation(IndexOperation::Settings {
index_uid, index_uid,
settings, settings,
allow_index_creation,
tasks, tasks,
}) => (index_uid, settings, tasks), }) => (index_uid, settings, tasks),
_ => unreachable!(), _ => unreachable!(),
@ -235,6 +257,7 @@ impl IndexScheduler {
IndexOperation::DocumentClearAndSetting { IndexOperation::DocumentClearAndSetting {
index_uid, index_uid,
cleared_tasks, cleared_tasks,
allow_index_creation,
settings, settings,
settings_tasks, settings_tasks,
}, },
@ -243,18 +266,26 @@ impl IndexScheduler {
BatchKind::SettingsAndDocumentImport { BatchKind::SettingsAndDocumentImport {
settings_ids, settings_ids,
method, method,
allow_index_creation,
import_ids, import_ids,
} => { } => {
let settings = self.create_next_batch_index( let settings = self.create_next_batch_index(
rtxn, rtxn,
index_uid.clone(), index_uid.clone(),
BatchKind::Settings { settings_ids }, BatchKind::Settings {
settings_ids,
allow_index_creation,
},
)?; )?;
let document_import = self.create_next_batch_index( let document_import = self.create_next_batch_index(
rtxn, rtxn,
index_uid.clone(), index_uid.clone(),
BatchKind::DocumentImport { method, import_ids }, BatchKind::DocumentImport {
method,
allow_index_creation,
import_ids,
},
)?; )?;
match (document_import, settings) { match (document_import, settings) {
@ -276,6 +307,7 @@ impl IndexScheduler {
index_uid, index_uid,
primary_key, primary_key,
method, method,
allow_index_creation,
documents_counts, documents_counts,
content_files, content_files,
document_import_tasks, document_import_tasks,
@ -521,6 +553,7 @@ impl IndexScheduler {
index_uid: _, index_uid: _,
primary_key, primary_key,
method, method,
allow_index_creation,
documents_counts, documents_counts,
content_files, content_files,
mut tasks, mut tasks,
@ -628,6 +661,7 @@ impl IndexScheduler {
IndexOperation::Settings { IndexOperation::Settings {
index_uid: _, index_uid: _,
settings, settings,
allow_index_creation,
mut tasks, mut tasks,
} => { } => {
let indexer_config = self.index_mapper.indexer_config(); let indexer_config = self.index_mapper.indexer_config();
@ -652,6 +686,7 @@ impl IndexScheduler {
index_uid, index_uid,
primary_key, primary_key,
method, method,
allow_index_creation,
documents_counts, documents_counts,
content_files, content_files,
document_import_tasks, document_import_tasks,
@ -664,6 +699,7 @@ impl IndexScheduler {
IndexOperation::Settings { IndexOperation::Settings {
index_uid: index_uid.clone(), index_uid: index_uid.clone(),
settings, settings,
allow_index_creation,
tasks: settings_tasks, tasks: settings_tasks,
}, },
)?; )?;
@ -675,6 +711,7 @@ impl IndexScheduler {
index_uid, index_uid,
primary_key, primary_key,
method, method,
allow_index_creation,
documents_counts, documents_counts,
content_files, content_files,
tasks: document_import_tasks, tasks: document_import_tasks,
@ -689,6 +726,7 @@ impl IndexScheduler {
index_uid, index_uid,
cleared_tasks, cleared_tasks,
settings, settings,
allow_index_creation,
settings_tasks, settings_tasks,
} => { } => {
let mut import_tasks = self.apply_index_operation( let mut import_tasks = self.apply_index_operation(
@ -706,6 +744,7 @@ impl IndexScheduler {
IndexOperation::Settings { IndexOperation::Settings {
index_uid, index_uid,
settings, settings,
allow_index_creation,
tasks: settings_tasks, tasks: settings_tasks,
}, },
)?; )?;

2
index-scheduler/src/lib.rs
View File

@ -446,7 +446,7 @@ impl IndexScheduler {
mod tests { mod tests {
use big_s::S; use big_s::S;
use insta::*; use insta::*;
use milli::update::IndexDocumentsMethod::{self, ReplaceDocuments, UpdateDocuments}; use milli::update::IndexDocumentsMethod::ReplaceDocuments;
use tempfile::TempDir; use tempfile::TempDir;
use uuid::Uuid; use uuid::Uuid;

48
index-scheduler/src/task.rs
View File

@ -16,6 +16,7 @@ pub struct TaskView {
pub uid: TaskId, pub uid: TaskId,
pub index_uid: Option<String>, pub index_uid: Option<String>,
pub status: Status, pub status: Status,
// TODO use our own Kind for the user
#[serde(rename = "type")] #[serde(rename = "type")]
pub kind: Kind, pub kind: Kind,
@ -175,17 +176,21 @@ impl KindWithContent {
pub fn as_kind(&self) -> Kind { pub fn as_kind(&self) -> Kind {
match self { match self {
KindWithContent::DocumentImport { KindWithContent::DocumentImport {
method: IndexDocumentsMethod::ReplaceDocuments, method,
allow_index_creation,
.. ..
} => Kind::DocumentAddition, } => Kind::DocumentImport {
KindWithContent::DocumentImport { method: *method,
method: IndexDocumentsMethod::UpdateDocuments, allow_index_creation: *allow_index_creation,
.. },
} => Kind::DocumentUpdate,
KindWithContent::DocumentImport { .. } => unreachable!(),
KindWithContent::DocumentDeletion { .. } => Kind::DocumentDeletion, KindWithContent::DocumentDeletion { .. } => Kind::DocumentDeletion,
KindWithContent::DocumentClear { .. } => Kind::DocumentClear, KindWithContent::DocumentClear { .. } => Kind::DocumentClear,
KindWithContent::Settings { .. } => Kind::Settings, KindWithContent::Settings {
allow_index_creation,
..
} => Kind::Settings {
allow_index_creation: *allow_index_creation,
},
KindWithContent::IndexCreation { .. } => Kind::IndexCreation, KindWithContent::IndexCreation { .. } => Kind::IndexCreation,
KindWithContent::IndexDeletion { .. } => Kind::IndexDeletion, KindWithContent::IndexDeletion { .. } => Kind::IndexDeletion,
KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate, KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate,
@ -299,11 +304,15 @@ impl KindWithContent {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub enum Kind { pub enum Kind {
DocumentAddition, DocumentImport {
DocumentUpdate, method: IndexDocumentsMethod,
allow_index_creation: bool,
},
DocumentDeletion, DocumentDeletion,
DocumentClear, DocumentClear,
Settings, Settings {
allow_index_creation: bool,
},
IndexCreation, IndexCreation,
IndexDeletion, IndexDeletion,
IndexUpdate, IndexUpdate,
@ -318,11 +327,22 @@ impl FromStr for Kind {
fn from_str(s: &str) -> Result<Self, Self::Err> { fn from_str(s: &str) -> Result<Self, Self::Err> {
match s { match s {
"document_addition" => Ok(Kind::DocumentAddition), "document_addition" => Ok(Kind::DocumentImport {
"document_update" => Ok(Kind::DocumentUpdate), method: IndexDocumentsMethod::ReplaceDocuments,
// TODO this doesn't make sense
allow_index_creation: false,
}),
"document_update" => Ok(Kind::DocumentImport {
method: IndexDocumentsMethod::UpdateDocuments,
// TODO this doesn't make sense
allow_index_creation: false,
}),
"document_deletion" => Ok(Kind::DocumentDeletion), "document_deletion" => Ok(Kind::DocumentDeletion),
"document_clear" => Ok(Kind::DocumentClear), "document_clear" => Ok(Kind::DocumentClear),
"settings" => Ok(Kind::Settings), "settings" => Ok(Kind::Settings {
// TODO this doesn't make sense
allow_index_creation: false,
}),
"index_creation" => Ok(Kind::IndexCreation), "index_creation" => Ok(Kind::IndexCreation),
"index_deletion" => Ok(Kind::IndexDeletion), "index_deletion" => Ok(Kind::IndexDeletion),
"index_update" => Ok(Kind::IndexUpdate), "index_update" => Ok(Kind::IndexUpdate),