diff --git a/index-scheduler/src/autobatcher.rs b/index-scheduler/src/autobatcher.rs index 91696351f..cae74c03c 100644 --- a/index-scheduler/src/autobatcher.rs +++ b/index-scheduler/src/autobatcher.rs @@ -10,6 +10,7 @@ pub enum BatchKind { }, DocumentImport { method: IndexDocumentsMethod, + allow_index_creation: bool, import_ids: Vec, }, DocumentDeletion { @@ -17,14 +18,17 @@ pub enum BatchKind { }, ClearAndSettings { other: Vec, + allow_index_creation: bool, settings_ids: Vec, }, SettingsAndDocumentImport { settings_ids: Vec, method: IndexDocumentsMethod, + allow_index_creation: bool, import_ids: Vec, }, Settings { + allow_index_creation: bool, settings_ids: Vec, }, IndexDeletion { @@ -50,18 +54,21 @@ impl BatchKind { Kind::IndexUpdate => Break(BatchKind::IndexUpdate { id: task_id }), Kind::IndexSwap => Break(BatchKind::IndexSwap { id: task_id }), Kind::DocumentClear => Continue(BatchKind::DocumentClear { ids: vec![task_id] }), - Kind::DocumentAddition => Continue(BatchKind::DocumentImport { - method: ReplaceDocuments, - import_ids: vec![task_id], - }), - Kind::DocumentUpdate => Continue(BatchKind::DocumentImport { - method: UpdateDocuments, + Kind::DocumentImport { + method, + allow_index_creation, + } => Continue(BatchKind::DocumentImport { + method, + allow_index_creation, import_ids: vec![task_id], }), Kind::DocumentDeletion => Continue(BatchKind::DocumentDeletion { deletion_ids: vec![task_id], }), - Kind::Settings => Continue(BatchKind::Settings { + Kind::Settings { + allow_index_creation, + } => Continue(BatchKind::Settings { + allow_index_creation, settings_ids: vec![task_id], }), Kind::DumpExport | Kind::Snapshot | Kind::CancelTask => unreachable!(), @@ -69,6 +76,7 @@ impl BatchKind { } /// Returns a `ControlFlow::Break` if you must stop right now. + #[rustfmt::skip] fn accumulate(self, id: TaskId, kind: Kind) -> ControlFlow { match (self, kind) { // We don't batch any of these operations @@ -76,31 +84,17 @@ impl BatchKind { // The index deletion can batch with everything but must stop after ( BatchKind::DocumentClear { mut ids } - | BatchKind::DocumentImport { - method: _, - import_ids: mut ids, - } - | BatchKind::DocumentDeletion { - deletion_ids: mut ids, - } - | BatchKind::Settings { - settings_ids: mut ids, - }, + | BatchKind::DocumentDeletion { deletion_ids: mut ids } + | BatchKind::DocumentImport { method: _, allow_index_creation: _, import_ids: mut ids } + | BatchKind::Settings { allow_index_creation: _, settings_ids: mut ids }, Kind::IndexDeletion, ) => { ids.push(id); Break(BatchKind::IndexDeletion { ids }) } ( - BatchKind::ClearAndSettings { - settings_ids: mut ids, - mut other, - } - | BatchKind::SettingsAndDocumentImport { - import_ids: mut ids, - method: _, - settings_ids: mut other, - }, + BatchKind::ClearAndSettings { settings_ids: mut ids, allow_index_creation: _, mut other } + | BatchKind::SettingsAndDocumentImport { import_ids: mut ids, method: _, allow_index_creation: _, settings_ids: mut other }, Kind::IndexDeletion, ) => { ids.push(id); @@ -117,110 +111,125 @@ impl BatchKind { } ( this @ BatchKind::DocumentClear { .. }, - Kind::DocumentAddition | Kind::DocumentUpdate | Kind::Settings, + Kind::DocumentImport { .. } | Kind::Settings { .. }, ) => Break(this), ( - BatchKind::DocumentImport { - method: _, - import_ids: mut ids, - }, + BatchKind::DocumentImport { method: _, allow_index_creation: _, import_ids: mut ids }, Kind::DocumentClear, ) => { ids.push(id); Continue(BatchKind::DocumentClear { ids }) } + // We only want to batch together document imports that are allowed to create the index + // or document imports not allowed to create an index if the first operation can. + ( + this @ BatchKind::DocumentImport { method: _, allow_index_creation: false, .. }, + Kind::DocumentImport { method: _, allow_index_creation: true }, + ) => Break(this), + // we can autobatch the same kind of document additions / updates ( - BatchKind::DocumentImport { - method: ReplaceDocuments, - mut import_ids, - }, - Kind::DocumentAddition, + BatchKind::DocumentImport { method: ReplaceDocuments, allow_index_creation, mut import_ids }, + Kind::DocumentImport { method: ReplaceDocuments, .. }, ) => { import_ids.push(id); Continue(BatchKind::DocumentImport { method: ReplaceDocuments, + allow_index_creation, import_ids, }) } ( - BatchKind::DocumentImport { - method: UpdateDocuments, - mut import_ids, - }, - Kind::DocumentUpdate, + BatchKind::DocumentImport { method: UpdateDocuments, allow_index_creation, mut import_ids }, + Kind::DocumentImport { method: UpdateDocuments, .. }, ) => { import_ids.push(id); Continue(BatchKind::DocumentImport { method: UpdateDocuments, + allow_index_creation, import_ids, }) } + // but we can't autobatch documents if it's not the same kind // this match branch MUST be AFTER the previous one ( this @ BatchKind::DocumentImport { .. }, - Kind::DocumentDeletion | Kind::DocumentAddition | Kind::DocumentUpdate, + Kind::DocumentDeletion | Kind::DocumentImport { .. }, ) => Break(this), - (BatchKind::DocumentImport { method, import_ids }, Kind::Settings) => { - Continue(BatchKind::SettingsAndDocumentImport { - settings_ids: vec![id], - method, - import_ids, - }) - } + + // We only want to batch together document imports that are allowed to create the index + // or document imports not allowed to create an index if the first operation can. + ( + this @ BatchKind::DocumentImport { allow_index_creation: false, .. }, + Kind::Settings { allow_index_creation: true }, + ) => Break(this), + ( + BatchKind::DocumentImport { method, allow_index_creation, import_ids }, + Kind::Settings { .. }, + ) => Continue(BatchKind::SettingsAndDocumentImport { + settings_ids: vec![id], + method, + allow_index_creation, + import_ids, + }), (BatchKind::DocumentDeletion { mut deletion_ids }, Kind::DocumentClear) => { deletion_ids.push(id); Continue(BatchKind::DocumentClear { ids: deletion_ids }) } - ( - this @ BatchKind::DocumentDeletion { .. }, - Kind::DocumentAddition | Kind::DocumentUpdate, - ) => Break(this), + (this @ BatchKind::DocumentDeletion { .. }, Kind::DocumentImport { .. }) => Break(this), (BatchKind::DocumentDeletion { mut deletion_ids }, Kind::DocumentDeletion) => { deletion_ids.push(id); Continue(BatchKind::DocumentDeletion { deletion_ids }) } - (this @ BatchKind::DocumentDeletion { .. }, Kind::Settings) => Break(this), + (this @ BatchKind::DocumentDeletion { .. }, Kind::Settings { .. }) => Break(this), - (BatchKind::Settings { settings_ids }, Kind::DocumentClear) => { - Continue(BatchKind::ClearAndSettings { - settings_ids: settings_ids, - other: vec![id], - }) - } + ( + BatchKind::Settings { settings_ids, allow_index_creation }, + Kind::DocumentClear, + ) => Continue(BatchKind::ClearAndSettings { + settings_ids: settings_ids, + allow_index_creation, + other: vec![id], + }), ( this @ BatchKind::Settings { .. }, - Kind::DocumentAddition | Kind::DocumentUpdate | Kind::DocumentDeletion, + Kind::DocumentImport { .. } | Kind::DocumentDeletion, ) => Break(this), - (BatchKind::Settings { mut settings_ids }, Kind::Settings) => { + ( + this @ BatchKind::Settings { allow_index_creation: false, .. }, + Kind::Settings { allow_index_creation: true }, + ) => Break(this), + ( + BatchKind::Settings { mut settings_ids, allow_index_creation }, + Kind::Settings { .. }, + ) => { settings_ids.push(id); - Continue(BatchKind::Settings { settings_ids }) + Continue(BatchKind::Settings { + allow_index_creation, + settings_ids, + }) } ( - BatchKind::ClearAndSettings { - mut other, - settings_ids, - }, + BatchKind::ClearAndSettings { mut other, settings_ids, allow_index_creation }, Kind::DocumentClear, ) => { other.push(id); Continue(BatchKind::ClearAndSettings { other, settings_ids, + allow_index_creation, }) } - ( - this @ BatchKind::ClearAndSettings { .. }, - Kind::DocumentAddition | Kind::DocumentUpdate, - ) => Break(this), + (this @ BatchKind::ClearAndSettings { .. }, Kind::DocumentImport { .. }) => Break(this), ( BatchKind::ClearAndSettings { mut other, settings_ids, + allow_index_creation, }, Kind::DocumentDeletion, ) => { @@ -228,64 +237,64 @@ impl BatchKind { Continue(BatchKind::ClearAndSettings { other, settings_ids, + allow_index_creation, }) } ( - BatchKind::ClearAndSettings { - mut settings_ids, - other, + this @ BatchKind::ClearAndSettings { allow_index_creation: false, .. }, + Kind::Settings { + allow_index_creation: true, }, - Kind::Settings, + ) => Break(this), + ( + BatchKind::ClearAndSettings { mut settings_ids, other, allow_index_creation }, + Kind::Settings { .. }, ) => { settings_ids.push(id); Continue(BatchKind::ClearAndSettings { other, settings_ids, + allow_index_creation, }) } ( - BatchKind::SettingsAndDocumentImport { - settings_ids, - method: _, - import_ids: mut other, - }, + BatchKind::SettingsAndDocumentImport { settings_ids, method: _, import_ids: mut other, allow_index_creation }, Kind::DocumentClear, ) => { other.push(id); Continue(BatchKind::ClearAndSettings { settings_ids, other, + allow_index_creation, }) } // we can batch the settings with a kind of document operation with the same kind of document operation ( - BatchKind::SettingsAndDocumentImport { - settings_ids, - method: ReplaceDocuments, - mut import_ids, - }, - Kind::DocumentAddition, + this @ BatchKind::SettingsAndDocumentImport { allow_index_creation: false, .. }, + Kind::DocumentImport { allow_index_creation: true, .. }, + ) => Break(this), + ( + BatchKind::SettingsAndDocumentImport { settings_ids, method: ReplaceDocuments, mut import_ids, allow_index_creation }, + Kind::DocumentImport { method: ReplaceDocuments, .. }, ) => { import_ids.push(id); Continue(BatchKind::SettingsAndDocumentImport { settings_ids, method: ReplaceDocuments, + allow_index_creation, import_ids, }) } ( - BatchKind::SettingsAndDocumentImport { - settings_ids, - method: UpdateDocuments, - mut import_ids, - }, - Kind::DocumentUpdate, + BatchKind::SettingsAndDocumentImport { settings_ids, method: UpdateDocuments, allow_index_creation, mut import_ids }, + Kind::DocumentImport { method: UpdateDocuments, .. }, ) => { import_ids.push(id); Continue(BatchKind::SettingsAndDocumentImport { settings_ids, method: UpdateDocuments, + allow_index_creation, import_ids, }) } @@ -293,20 +302,21 @@ impl BatchKind { // this MUST be AFTER the two previous branch ( this @ BatchKind::SettingsAndDocumentImport { .. }, - Kind::DocumentDeletion | Kind::DocumentAddition | Kind::DocumentUpdate, + Kind::DocumentDeletion | Kind::DocumentImport { .. }, ) => Break(this), ( - BatchKind::SettingsAndDocumentImport { - mut settings_ids, - method, - import_ids, - }, - Kind::Settings, + this @ BatchKind::SettingsAndDocumentImport { allow_index_creation: false, .. }, + Kind::Settings { allow_index_creation: true }, + ) => Break(this), + ( + BatchKind::SettingsAndDocumentImport { mut settings_ids, method, allow_index_creation, import_ids }, + Kind::Settings { .. }, ) => { settings_ids.push(id); Continue(BatchKind::SettingsAndDocumentImport { settings_ids, method, + allow_index_creation, import_ids, }) } @@ -362,119 +372,129 @@ mod tests { #[test] fn autobatch_simple_operation_together() { // we can autobatch one or multiple DocumentAddition together - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentAddition]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0, 1, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })"); // we can autobatch one or multiple DocumentUpdate together - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentUpdate]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0, 1, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 1, 2] })"); // we can autobatch one or multiple DocumentDeletion together assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion]), @"Some(DocumentDeletion { deletion_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentDeletion, DocumentDeletion]), @"Some(DocumentDeletion { deletion_ids: [0, 1, 2] })"); // we can autobatch one or multiple Settings together - assert_smol_debug_snapshot!(autobatch_from([Settings]), @"Some(Settings { settings_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([Settings, Settings, Settings]), @"Some(Settings { settings_ids: [0, 1, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }]), @"Some(Settings { allow_index_creation: true, settings_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }, Settings { allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(Settings { allow_index_creation: true, settings_ids: [0, 1, 2] })"); } #[test] fn simple_document_operation_dont_autobatch_with_other() { // addition, updates and deletion can't batch together - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentUpdate]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentDeletion]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentAddition]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentDeletion]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentAddition]), @"Some(DocumentDeletion { deletion_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentUpdate]), @"Some(DocumentDeletion { deletion_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentDeletion]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentDeletion]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentDeletion { deletion_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentDeletion { deletion_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexCreation]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexCreation]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexCreation]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexCreation]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexCreation]), @"Some(DocumentDeletion { deletion_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexUpdate]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexUpdate]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexUpdate]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexUpdate]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexUpdate]), @"Some(DocumentDeletion { deletion_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexSwap]), @"Some(DocumentImport { method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexSwap]), @"Some(DocumentImport { method: UpdateDocuments, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexSwap]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexSwap]), @"Some(DocumentImport { method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexSwap]), @"Some(DocumentDeletion { deletion_ids: [0] })"); } #[test] fn document_addition_batch_with_settings() { // simple case - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); // multiple settings and doc addition - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, Settings, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, import_ids: [0, 1] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, Settings, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, import_ids: [0, 1] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })"); // addition and setting unordered - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentAddition, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: ReplaceDocuments, import_ids: [0, 2] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentUpdate, Settings]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: UpdateDocuments, import_ids: [0, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1, 3], method: UpdateDocuments, allow_index_creation: true, import_ids: [0, 2] })"); // We ensure this kind of batch doesn't batch with forbidden operations - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentAddition]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, import_ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentDeletion]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexCreation]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexUpdate]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexSwap]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, import_ids: [0] })"); } #[test] fn clear_and_additions() { // these two doesn't need to batch - assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentAddition]), @"Some(DocumentClear { ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentUpdate]), @"Some(DocumentClear { ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentClear, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0] })"); // Basic use case - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2] })"); // This batch kind doesn't mix with other document addition - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentAddition]), @"Some(DocumentClear { ids: [0, 1, 2] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentUpdate]), @"Some(DocumentClear { ids: [0, 1, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentClear, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0, 1, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentClear, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(DocumentClear { ids: [0, 1, 2] })"); // But you can batch multiple clear together - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentImport { method: UpdateDocuments, allow_index_creation: true }, DocumentClear, DocumentClear, DocumentClear]), @"Some(DocumentClear { ids: [0, 1, 2, 3, 4] })"); } #[test] fn clear_and_additions_and_settings() { // A clear don't need to autobatch the settings that happens AFTER there is no documents - assert_smol_debug_snapshot!(autobatch_from([DocumentClear, Settings]), @"Some(DocumentClear { ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentClear, Settings { allow_index_creation: true }]), @"Some(DocumentClear { ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([Settings, DocumentClear, Settings]), @"Some(ClearAndSettings { other: [1], settings_ids: [0, 2] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], settings_ids: [1] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], settings_ids: [1] })"); + assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }, DocumentClear, Settings { allow_index_creation: true }]), @"Some(ClearAndSettings { other: [1], allow_index_creation: true, settings_ids: [0, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear]), @"Some(ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] })"); } #[test] fn anything_and_index_deletion() { // The indexdeletion doesn't batch with anything that happens AFTER - assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentAddition]), @"Some(IndexDeletion { ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentUpdate]), @"Some(IndexDeletion { ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(IndexDeletion { ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentImport { method: UpdateDocuments, allow_index_creation: true }]), @"Some(IndexDeletion { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentDeletion]), @"Some(IndexDeletion { ids: [0] })"); assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, DocumentClear]), @"Some(IndexDeletion { ids: [0] })"); - assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, Settings]), @"Some(IndexDeletion { ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([IndexDeletion, Settings { allow_index_creation: true }]), @"Some(IndexDeletion { ids: [0] })"); // The index deletion can accept almost any type of BatchKind and transform it to an IndexDeletion // First, the basic cases - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentDeletion, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); assert_smol_debug_snapshot!(autobatch_from([DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); - assert_smol_debug_snapshot!(autobatch_from([Settings, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); + assert_smol_debug_snapshot!(autobatch_from([Settings { allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 1] })"); // Then the mixed cases - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentAddition, Settings, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })"); - assert_smol_debug_snapshot!(autobatch_from([DocumentUpdate, Settings, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, IndexDeletion]), @"Some(IndexDeletion { ids: [0, 2, 1] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: UpdateDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }, DocumentClear, IndexDeletion]), @"Some(IndexDeletion { ids: [1, 3, 0, 2] })"); + } + + #[test] + fn allowed_and_disallowed_index_creation() { + // DocumentImport that can create indexes can't be mixed with those disallowed to do so + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: false }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, DocumentImport { method: ReplaceDocuments, allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: true, import_ids: [0, 1] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: false }, DocumentImport { method: ReplaceDocuments, allow_index_creation: false }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0, 1] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: true }, Settings { allow_index_creation: true }]), @"Some(SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, import_ids: [0] })"); + assert_smol_debug_snapshot!(autobatch_from([DocumentImport { method: ReplaceDocuments, allow_index_creation: false }, Settings { allow_index_creation: true }]), @"Some(DocumentImport { method: ReplaceDocuments, allow_index_creation: false, import_ids: [0] })"); } } diff --git a/index-scheduler/src/batch.rs b/index-scheduler/src/batch.rs index 97dd9b7a1..a344a7cdd 100644 --- a/index-scheduler/src/batch.rs +++ b/index-scheduler/src/batch.rs @@ -4,7 +4,7 @@ use crate::{ Error, IndexScheduler, Result, TaskId, }; use index::apply_settings_to_builder; -use index::error::{IndexError, MilliError}; +use index::error::IndexError; use index::{Settings, Unchecked}; use log::{debug, info}; use milli::documents::DocumentsBatchReader; @@ -39,6 +39,7 @@ pub(crate) enum IndexOperation { index_uid: String, primary_key: Option, method: IndexDocumentsMethod, + allow_index_creation: bool, documents_counts: Vec, content_files: Vec, tasks: Vec, @@ -56,6 +57,7 @@ pub(crate) enum IndexOperation { index_uid: String, // TODO what's that boolean, does it mean that it removes things or what? settings: Vec<(bool, Settings)>, + allow_index_creation: bool, tasks: Vec, }, DocumentClearAndSetting { @@ -64,6 +66,7 @@ pub(crate) enum IndexOperation { // TODO what's that boolean, does it mean that it removes things or what? settings: Vec<(bool, Settings)>, + allow_index_creation: bool, settings_tasks: Vec, }, SettingsAndDocumentImport { @@ -71,6 +74,7 @@ pub(crate) enum IndexOperation { primary_key: Option, method: IndexDocumentsMethod, + allow_index_creation: bool, documents_counts: Vec, content_files: Vec, document_import_tasks: Vec, @@ -126,7 +130,11 @@ impl IndexScheduler { index_uid, }))) } - BatchKind::DocumentImport { method, import_ids } => { + BatchKind::DocumentImport { + method, + import_ids, + allow_index_creation, + } => { let tasks = self.get_existing_tasks(rtxn, import_ids)?; let primary_key = match &tasks[0].kind { KindWithContent::DocumentImport { primary_key, .. } => primary_key.clone(), @@ -154,6 +162,7 @@ impl IndexScheduler { index_uid, primary_key, method, + allow_index_creation, documents_counts, content_files, tasks, @@ -181,7 +190,10 @@ impl IndexScheduler { }, ))) } - BatchKind::Settings { settings_ids } => { + BatchKind::Settings { + settings_ids, + allow_index_creation, + } => { let tasks = self.get_existing_tasks(rtxn, settings_ids)?; let mut settings = Vec::new(); @@ -199,20 +211,30 @@ impl IndexScheduler { Ok(Some(Batch::IndexOperation(IndexOperation::Settings { index_uid, settings, + allow_index_creation, tasks, }))) } BatchKind::ClearAndSettings { other, settings_ids, + allow_index_creation, } => { let (index_uid, settings, settings_tasks) = match self - .create_next_batch_index(rtxn, index_uid, BatchKind::Settings { settings_ids })? + .create_next_batch_index( + rtxn, + index_uid, + BatchKind::Settings { + settings_ids, + allow_index_creation, + }, + )? .unwrap() { Batch::IndexOperation(IndexOperation::Settings { index_uid, settings, + allow_index_creation, tasks, }) => (index_uid, settings, tasks), _ => unreachable!(), @@ -235,6 +257,7 @@ impl IndexScheduler { IndexOperation::DocumentClearAndSetting { index_uid, cleared_tasks, + allow_index_creation, settings, settings_tasks, }, @@ -243,18 +266,26 @@ impl IndexScheduler { BatchKind::SettingsAndDocumentImport { settings_ids, method, + allow_index_creation, import_ids, } => { let settings = self.create_next_batch_index( rtxn, index_uid.clone(), - BatchKind::Settings { settings_ids }, + BatchKind::Settings { + settings_ids, + allow_index_creation, + }, )?; let document_import = self.create_next_batch_index( rtxn, index_uid.clone(), - BatchKind::DocumentImport { method, import_ids }, + BatchKind::DocumentImport { + method, + allow_index_creation, + import_ids, + }, )?; match (document_import, settings) { @@ -276,6 +307,7 @@ impl IndexScheduler { index_uid, primary_key, method, + allow_index_creation, documents_counts, content_files, document_import_tasks, @@ -521,6 +553,7 @@ impl IndexScheduler { index_uid: _, primary_key, method, + allow_index_creation, documents_counts, content_files, mut tasks, @@ -628,6 +661,7 @@ impl IndexScheduler { IndexOperation::Settings { index_uid: _, settings, + allow_index_creation, mut tasks, } => { let indexer_config = self.index_mapper.indexer_config(); @@ -652,6 +686,7 @@ impl IndexScheduler { index_uid, primary_key, method, + allow_index_creation, documents_counts, content_files, document_import_tasks, @@ -664,6 +699,7 @@ impl IndexScheduler { IndexOperation::Settings { index_uid: index_uid.clone(), settings, + allow_index_creation, tasks: settings_tasks, }, )?; @@ -675,6 +711,7 @@ impl IndexScheduler { index_uid, primary_key, method, + allow_index_creation, documents_counts, content_files, tasks: document_import_tasks, @@ -689,6 +726,7 @@ impl IndexScheduler { index_uid, cleared_tasks, settings, + allow_index_creation, settings_tasks, } => { let mut import_tasks = self.apply_index_operation( @@ -706,6 +744,7 @@ impl IndexScheduler { IndexOperation::Settings { index_uid, settings, + allow_index_creation, tasks: settings_tasks, }, )?; diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index d8a06eca6..7f1ba3d5b 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -446,7 +446,7 @@ impl IndexScheduler { mod tests { use big_s::S; use insta::*; - use milli::update::IndexDocumentsMethod::{self, ReplaceDocuments, UpdateDocuments}; + use milli::update::IndexDocumentsMethod::ReplaceDocuments; use tempfile::TempDir; use uuid::Uuid; diff --git a/index-scheduler/src/task.rs b/index-scheduler/src/task.rs index a4dba9224..4e08b70bc 100644 --- a/index-scheduler/src/task.rs +++ b/index-scheduler/src/task.rs @@ -16,6 +16,7 @@ pub struct TaskView { pub uid: TaskId, pub index_uid: Option, pub status: Status, + // TODO use our own Kind for the user #[serde(rename = "type")] pub kind: Kind, @@ -175,17 +176,21 @@ impl KindWithContent { pub fn as_kind(&self) -> Kind { match self { KindWithContent::DocumentImport { - method: IndexDocumentsMethod::ReplaceDocuments, + method, + allow_index_creation, .. - } => Kind::DocumentAddition, - KindWithContent::DocumentImport { - method: IndexDocumentsMethod::UpdateDocuments, - .. - } => Kind::DocumentUpdate, - KindWithContent::DocumentImport { .. } => unreachable!(), + } => Kind::DocumentImport { + method: *method, + allow_index_creation: *allow_index_creation, + }, KindWithContent::DocumentDeletion { .. } => Kind::DocumentDeletion, KindWithContent::DocumentClear { .. } => Kind::DocumentClear, - KindWithContent::Settings { .. } => Kind::Settings, + KindWithContent::Settings { + allow_index_creation, + .. + } => Kind::Settings { + allow_index_creation: *allow_index_creation, + }, KindWithContent::IndexCreation { .. } => Kind::IndexCreation, KindWithContent::IndexDeletion { .. } => Kind::IndexDeletion, KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate, @@ -299,11 +304,15 @@ impl KindWithContent { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum Kind { - DocumentAddition, - DocumentUpdate, + DocumentImport { + method: IndexDocumentsMethod, + allow_index_creation: bool, + }, DocumentDeletion, DocumentClear, - Settings, + Settings { + allow_index_creation: bool, + }, IndexCreation, IndexDeletion, IndexUpdate, @@ -318,11 +327,22 @@ impl FromStr for Kind { fn from_str(s: &str) -> Result { match s { - "document_addition" => Ok(Kind::DocumentAddition), - "document_update" => Ok(Kind::DocumentUpdate), + "document_addition" => Ok(Kind::DocumentImport { + method: IndexDocumentsMethod::ReplaceDocuments, + // TODO this doesn't make sense + allow_index_creation: false, + }), + "document_update" => Ok(Kind::DocumentImport { + method: IndexDocumentsMethod::UpdateDocuments, + // TODO this doesn't make sense + allow_index_creation: false, + }), "document_deletion" => Ok(Kind::DocumentDeletion), "document_clear" => Ok(Kind::DocumentClear), - "settings" => Ok(Kind::Settings), + "settings" => Ok(Kind::Settings { + // TODO this doesn't make sense + allow_index_creation: false, + }), "index_creation" => Ok(Kind::IndexCreation), "index_deletion" => Ok(Kind::IndexDeletion), "index_update" => Ok(Kind::IndexUpdate),