diff --git a/crates/index-scheduler/src/queue/test.rs b/crates/index-scheduler/src/queue/test.rs index 5a886b088..eb3314496 100644 --- a/crates/index-scheduler/src/queue/test.rs +++ b/crates/index-scheduler/src/queue/test.rs @@ -165,6 +165,7 @@ fn test_disable_auto_deletion_of_tasks() { let (index_scheduler, mut handle) = IndexScheduler::test_with_custom_config(vec![], |config| { config.cleanup_enabled = false; config.max_number_of_tasks = 2; + None }); index_scheduler @@ -228,6 +229,7 @@ fn test_disable_auto_deletion_of_tasks() { fn test_auto_deletion_of_tasks() { let (index_scheduler, mut handle) = IndexScheduler::test_with_custom_config(vec![], |config| { config.max_number_of_tasks = 2; + None }); index_scheduler @@ -325,6 +327,7 @@ fn test_task_queue_is_full() { let (index_scheduler, mut handle) = IndexScheduler::test_with_custom_config(vec![], |config| { // that's the minimum map size possible config.task_db_size = 1048576; + None }); index_scheduler diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index 7b4775704..3ad2076c8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -7,8 +7,8 @@ snapshot_kind: text [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 0, status: succeeded, details: { from: v1.12.0 }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} @@ -19,11 +19,11 @@ succeeded [0,1,2,4,] failed [3,] ---------------------------------------------------------------------- ### Kind: -"indexCreation" [0,2,3,4,] -"upgradeDatabase" [1,] +"indexCreation" [1,2,3,4,] +"upgradeDatabase" [0,] ---------------------------------------------------------------------- ### Index Tasks: -catto [0,] +catto [1,] doggo [2,3,] girafo [4,] ---------------------------------------------------------------------- @@ -44,29 +44,29 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### Started At: -[timestamp] [1,] [timestamp] [0,] +[timestamp] [1,] [timestamp] [2,] [timestamp] [3,] [timestamp] [4,] ---------------------------------------------------------------------- ### Finished At: -[timestamp] [1,] [timestamp] [0,] +[timestamp] [1,] [timestamp] [2,] [timestamp] [3,] [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.13.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } 4 {uid: 4, details: {"primaryKey":"leaves"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"girafo":1}}, } ---------------------------------------------------------------------- ### Batch to tasks mapping: -0 [1,] -1 [0,] +0 [0,] +1 [1,] 2 [2,] 3 [3,] 4 [4,] @@ -85,8 +85,8 @@ doggo [2,3,] girafo [4,] ---------------------------------------------------------------------- ### Batches Enqueued At: -[timestamp] [1,] [timestamp] [0,] +[timestamp] [1,] [timestamp] [2,] [timestamp] [3,] [timestamp] [4,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap similarity index 90% rename from crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade.snap rename to crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap index 8064f8425..9e490843e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap @@ -7,24 +7,24 @@ snapshot_kind: text [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} -5 {uid: 5, batch_uid: 5, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "test" }, kind: TaskDeletion { query: "test", tasks: RoaringBitmap<[1]> }} +5 {uid: 5, batch_uid: 5, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "types=upgradeDatabase" }, kind: TaskDeletion { query: "types=upgradeDatabase", tasks: RoaringBitmap<[0]> }} ---------------------------------------------------------------------- ### Status: enqueued [] -succeeded [0,2,4,5,] +succeeded [1,2,4,5,] failed [3,] ---------------------------------------------------------------------- ### Kind: -"indexCreation" [0,2,3,4,] +"indexCreation" [1,2,3,4,] "taskDeletion" [5,] "upgradeDatabase" [] ---------------------------------------------------------------------- ### Index Tasks: -catto [0,] +catto [1,] doggo [2,3,] girafo [4,] ---------------------------------------------------------------------- @@ -38,21 +38,21 @@ girafo: { number_of_documents: 0, field_distribution: {} } ---------------------------------------------------------------------- ### Enqueued At: -[timestamp] [0,] +[timestamp] [1,] [timestamp] [2,] [timestamp] [3,] [timestamp] [4,] [timestamp] [5,] ---------------------------------------------------------------------- ### Started At: -[timestamp] [0,] +[timestamp] [1,] [timestamp] [2,] [timestamp] [3,] [timestamp] [4,] [timestamp] [5,] ---------------------------------------------------------------------- ### Finished At: -[timestamp] [0,] +[timestamp] [1,] [timestamp] [2,] [timestamp] [3,] [timestamp] [4,] @@ -63,10 +63,10 @@ girafo: { number_of_documents: 0, field_distribution: {} } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, } 4 {uid: 4, details: {"primaryKey":"leaves"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"girafo":1}}, } -5 {uid: 5, details: {"matchedTasks":1,"deletedTasks":1,"originalFilter":"test"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"taskDeletion":1},"indexUids":{}}, } +5 {uid: 5, details: {"matchedTasks":1,"deletedTasks":1,"originalFilter":"types=upgradeDatabase"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"taskDeletion":1},"indexUids":{}}, } ---------------------------------------------------------------------- ### Batch to tasks mapping: -1 [0,] +1 [1,] 2 [2,] 3 [3,] 4 [4,] @@ -87,8 +87,8 @@ doggo [2,3,] girafo [4,] ---------------------------------------------------------------------- ### Batches Enqueued At: -[timestamp] [1,] [timestamp] [0,] +[timestamp] [1,] [timestamp] [2,] [timestamp] [3,] [timestamp] [4,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap new file mode 100644 index 000000000..eead6e773 --- /dev/null +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -0,0 +1,51 @@ +--- +source: crates/index-scheduler/src/scheduler/test_failure.rs +snapshot_kind: text +--- +### Autobatching Enabled = true +### Processing batch None: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +---------------------------------------------------------------------- +### Status: +enqueued [0,] +---------------------------------------------------------------------- +### Kind: +"upgradeDatabase" [0,] +---------------------------------------------------------------------- +### Index Tasks: +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### All Batches: +---------------------------------------------------------------------- +### Batch to tasks mapping: +---------------------------------------------------------------------- +### Batches Status: +---------------------------------------------------------------------- +### Batches Kind: +---------------------------------------------------------------------- +### Batches Index Tasks: +---------------------------------------------------------------------- +### Batches Enqueued At: +---------------------------------------------------------------------- +### Batches Started At: +---------------------------------------------------------------------- +### Batches Finished At: +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_and_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap similarity index 91% rename from crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_and_upgrade_task.snap rename to crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index afad39e32..52f0b61a7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_and_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -7,18 +7,18 @@ snapshot_kind: text [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] ---------------------------------------------------------------------- ### Kind: -"indexCreation" [0,] -"upgradeDatabase" [1,] +"indexCreation" [1,] +"upgradeDatabase" [0,] ---------------------------------------------------------------------- ### Index Tasks: -catto [0,] +catto [1,] ---------------------------------------------------------------------- ### Index Mapper: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 6294cbbc3..96efafc9e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -7,19 +7,19 @@ snapshot_kind: text [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: v1.12.0 }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: -enqueued [0,] -failed [1,] +enqueued [1,] +failed [0,] ---------------------------------------------------------------------- ### Kind: -"indexCreation" [0,] -"upgradeDatabase" [1,] +"indexCreation" [1,] +"upgradeDatabase" [0,] ---------------------------------------------------------------------- ### Index Tasks: -catto [0,] +catto [1,] ---------------------------------------------------------------------- ### Index Mapper: @@ -32,16 +32,16 @@ catto [0,] [timestamp] [1,] ---------------------------------------------------------------------- ### Started At: -[timestamp] [1,] +[timestamp] [0,] ---------------------------------------------------------------------- ### Finished At: -[timestamp] [1,] +[timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.13.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } ---------------------------------------------------------------------- ### Batch to tasks mapping: -0 [1,] +0 [0,] ---------------------------------------------------------------------- ### Batches Status: failed [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 91e6b80f2..bd223298d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -7,20 +7,20 @@ snapshot_kind: text [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: v1.12.0 }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: -enqueued [0,2,] -failed [1,] +enqueued [1,2,] +failed [0,] ---------------------------------------------------------------------- ### Kind: -"indexCreation" [0,2,] -"upgradeDatabase" [1,] +"indexCreation" [1,2,] +"upgradeDatabase" [0,] ---------------------------------------------------------------------- ### Index Tasks: -catto [0,] +catto [1,] doggo [2,] ---------------------------------------------------------------------- ### Index Mapper: @@ -35,16 +35,16 @@ doggo [2,] [timestamp] [2,] ---------------------------------------------------------------------- ### Started At: -[timestamp] [1,] +[timestamp] [0,] ---------------------------------------------------------------------- ### Finished At: -[timestamp] [1,] +[timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.13.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } ---------------------------------------------------------------------- ### Batch to tasks mapping: -0 [1,] +0 [0,] ---------------------------------------------------------------------- ### Batches Status: failed [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 940e68be0..5bb2d57cf 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -7,22 +7,22 @@ snapshot_kind: text [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 0, status: succeeded, details: { from: v1.12.0 }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 13, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: -enqueued [0,2,3,] -succeeded [1,] +enqueued [1,2,3,] +succeeded [0,] failed [] ---------------------------------------------------------------------- ### Kind: -"indexCreation" [0,2,3,] -"upgradeDatabase" [1,] +"indexCreation" [1,2,3,] +"upgradeDatabase" [0,] ---------------------------------------------------------------------- ### Index Tasks: -catto [0,] +catto [1,] doggo [2,3,] ---------------------------------------------------------------------- ### Index Mapper: @@ -38,16 +38,16 @@ doggo [2,3,] [timestamp] [3,] ---------------------------------------------------------------------- ### Started At: -[timestamp] [1,] +[timestamp] [0,] ---------------------------------------------------------------------- ### Finished At: -[timestamp] [1,] +[timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.13.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, } ---------------------------------------------------------------------- ### Batch to tasks mapping: -0 [1,] +0 [0,] ---------------------------------------------------------------------- ### Batches Status: succeeded [0,] diff --git a/crates/index-scheduler/src/scheduler/test_failure.rs b/crates/index-scheduler/src/scheduler/test_failure.rs index 3b895ef2f..dba0bb072 100644 --- a/crates/index-scheduler/src/scheduler/test_failure.rs +++ b/crates/index-scheduler/src/scheduler/test_failure.rs @@ -6,6 +6,7 @@ use meili_snap::snapshot; use meilisearch_types::milli::obkv_to_json; use meilisearch_types::milli::update::IndexDocumentsMethod::*; use meilisearch_types::milli::update::Setting; +use meilisearch_types::tasks::Kind; use meilisearch_types::tasks::KindWithContent; use roaring::RoaringBitmap; @@ -253,15 +254,16 @@ fn panic_in_process_batch_for_index_creation() { #[test] fn upgrade_failure() { + // By starting the index-scheduler at the v1.12.0 an upgrade task should be automatically enqueued let (index_scheduler, mut handle) = - IndexScheduler::test(true, vec![(1, FailureLocation::ProcessUpgrade)]); + IndexScheduler::test_with_custom_config(vec![(1, FailureLocation::ProcessUpgrade)], |_| { + Some((1, 12, 0)) + }); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "register_automatic_upgrade_task"); let kind = index_creation_task("catto", "mouse"); let _task = index_scheduler.register(kind, None, false).unwrap(); - let upgrade_database_task = index_scheduler - .register(KindWithContent::UpgradeDatabase { from: (1, 12, 0) }, None, false) - .unwrap(); - snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task_and_upgrade_task"); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task_while_the_upgrade_task_is_enqueued"); handle.advance_one_failed_batch(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "upgrade_task_failed"); @@ -275,7 +277,9 @@ fn upgrade_failure() { // =====> After a restart is it still working as expected? let (index_scheduler, mut handle) = - handle.restart(index_scheduler, true, vec![(1, FailureLocation::ProcessUpgrade)]); + handle.restart(index_scheduler, true, vec![(1, FailureLocation::ProcessUpgrade)], |_| { + Some((1, 12, 0)) // the upgrade task should be rerun automatically and nothing else should be enqueued + }); handle.advance_one_failed_batch(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "upgrade_task_failed_again"); @@ -286,7 +290,8 @@ fn upgrade_failure() { handle.scheduler_is_down(); // =====> After a rerestart and without failure can we upgrade the indexes and process the tasks - let (index_scheduler, mut handle) = handle.restart(index_scheduler, true, vec![]); + let (index_scheduler, mut handle) = + handle.restart(index_scheduler, true, vec![], |_| Some((1, 12, 0))); handle.advance_one_successful_batch(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "upgrade_task_succeeded"); @@ -300,12 +305,18 @@ fn upgrade_failure() { handle.advance_one_successful_batch(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_processing_everything"); + let (upgrade_tasks_ids, _) = index_scheduler + .get_task_ids_from_authorized_indexes( + &crate::Query { types: Some(vec![Kind::UpgradeDatabase]), ..Default::default() }, + &Default::default(), + ) + .unwrap(); // When deleting the single upgrade task it should remove the associated batch let _task = index_scheduler .register( KindWithContent::TaskDeletion { - query: String::from("test"), - tasks: RoaringBitmap::from_iter([upgrade_database_task.uid]), + query: String::from("types=upgradeDatabase"), + tasks: upgrade_tasks_ids, }, None, false, @@ -313,5 +324,5 @@ fn upgrade_failure() { .unwrap(); handle.advance_one_successful_batch(); - snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_removing_the_upgrade"); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_removing_the_upgrade_tasks"); } diff --git a/crates/index-scheduler/src/test_utils.rs b/crates/index-scheduler/src/test_utils.rs index b1e44e32c..072220b6c 100644 --- a/crates/index-scheduler/src/test_utils.rs +++ b/crates/index-scheduler/src/test_utils.rs @@ -75,12 +75,13 @@ impl IndexScheduler { ) -> (Self, IndexSchedulerHandle) { Self::test_with_custom_config(planned_failures, |config| { config.autobatching_enabled = autobatching_enabled; + None }) } pub(crate) fn test_with_custom_config( planned_failures: Vec<(usize, FailureLocation)>, - configuration: impl Fn(&mut IndexSchedulerOptions), + configuration: impl Fn(&mut IndexSchedulerOptions) -> Option<(u32, u32, u32)>, ) -> (Self, IndexSchedulerHandle) { let tempdir = TempDir::new().unwrap(); let (sender, receiver) = crossbeam_channel::bounded(0); @@ -111,13 +112,13 @@ impl IndexScheduler { instance_features: Default::default(), auto_upgrade: true, // Don't cost much and will ensure the happy path works }; - configuration(&mut options); - - let version = ( - versioning::VERSION_MAJOR.parse().unwrap(), - versioning::VERSION_MINOR.parse().unwrap(), - versioning::VERSION_PATCH.parse().unwrap(), - ); + let version = configuration(&mut options).unwrap_or_else(|| { + ( + versioning::VERSION_MAJOR.parse().unwrap(), + versioning::VERSION_MINOR.parse().unwrap(), + versioning::VERSION_PATCH.parse().unwrap(), + ) + }); let index_scheduler = Self::new(options, version, sender, planned_failures).unwrap(); @@ -241,6 +242,7 @@ impl IndexSchedulerHandle { index_scheduler: IndexScheduler, autobatching_enabled: bool, planned_failures: Vec<(usize, FailureLocation)>, + configuration: impl Fn(&mut IndexSchedulerOptions) -> Option<(u32, u32, u32)>, ) -> (IndexScheduler, Self) { drop(index_scheduler); let Self { _tempdir: tempdir, index_scheduler, test_breakpoint_rcv, last_breakpoint: _ } = @@ -264,6 +266,7 @@ impl IndexSchedulerHandle { let (scheduler, mut handle) = IndexScheduler::test_with_custom_config(planned_failures, |config| { + let version = configuration(config); config.autobatching_enabled = autobatching_enabled; config.version_file_path = tempdir.path().join(VERSION_FILE_NAME); config.auth_path = tempdir.path().join("auth"); @@ -272,6 +275,7 @@ impl IndexSchedulerHandle { config.indexes_path = tempdir.path().join("indexes"); config.snapshots_path = tempdir.path().join("snapshots"); config.dumps_path = tempdir.path().join("dumps"); + version }); handle._tempdir = tempdir; (scheduler, handle)