From d09b771bceca95e20bbcca2f218b068f09908fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20Bour?= Date: Mon, 3 Apr 2023 21:08:47 +0200 Subject: [PATCH] Add a dedicated error variant for planned failures in index scheduler tests Fixes #3086 --- index-scheduler/src/error.rs | 45 +++++++++++++++++++ index-scheduler/src/lib.rs | 12 ++--- .../document_addition_failed.snap | 2 +- .../index_creation_failed.snap | 2 +- .../lib.rs/query_tasks_simple/end.snap | 2 +- 5 files changed, 51 insertions(+), 12 deletions(-) diff --git a/index-scheduler/src/error.rs b/index-scheduler/src/error.rs index 7b884e0a4..3a19ed4d2 100644 --- a/index-scheduler/src/error.rs +++ b/index-scheduler/src/error.rs @@ -134,6 +134,48 @@ pub enum Error { TaskDatabaseUpdate(Box), #[error(transparent)] HeedTransaction(heed::Error), + + #[cfg(test)] + #[error("Planned failure for tests.")] + PlannedFailure, +} + +impl Error { + pub fn is_recoverable(&self) -> bool { + match self { + Error::IndexNotFound(_) + | Error::IndexAlreadyExists(_) + | Error::SwapDuplicateIndexFound(_) + | Error::SwapDuplicateIndexesFound(_) + | Error::SwapIndexNotFound(_) + | Error::NoSpaceLeftInTaskQueue + | Error::SwapIndexesNotFound(_) + | Error::CorruptedDump + | Error::InvalidTaskDate { .. } + | Error::InvalidTaskUids { .. } + | Error::InvalidTaskStatuses { .. } + | Error::InvalidTaskTypes { .. } + | Error::InvalidTaskCanceledBy { .. } + | Error::InvalidIndexUid { .. } + | Error::TaskNotFound(_) + | Error::TaskDeletionWithEmptyQuery + | Error::TaskCancelationWithEmptyQuery + | Error::Dump(_) + | Error::Heed(_) + | Error::Milli(_) + | Error::ProcessBatchPanicked + | Error::FileStore(_) + | Error::IoError(_) + | Error::Persist(_) + | Error::Anyhow(_) => true, + Error::CreateBatch(_) + | Error::CorruptedTaskQueue + | Error::TaskDatabaseUpdate(_) + | Error::HeedTransaction(_) => false, + #[cfg(test)] + Error::PlannedFailure => false, + } + } } impl ErrorCode for Error { @@ -171,6 +213,9 @@ impl ErrorCode for Error { Error::CorruptedDump => Code::Internal, Error::TaskDatabaseUpdate(_) => Code::Internal, Error::CreateBatch(_) => Code::Internal, + + #[cfg(test)] + Error::PlannedFailure => Code::Internal, } } } diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index d713fca17..e5c698690 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -530,13 +530,7 @@ impl IndexScheduler { Err(e) => { log::error!("{}", e); // Wait one second when an irrecoverable error occurs. - if matches!( - e, - Error::CorruptedTaskQueue - | Error::TaskDatabaseUpdate(_) - | Error::HeedTransaction(_) - | Error::CreateBatch(_) - ) { + if !e.is_recoverable() { std::thread::sleep(Duration::from_secs(1)); } } @@ -1426,7 +1420,7 @@ mod tests { (index_scheduler, index_scheduler_handle) } - /// Return a [`CorruptedTaskQueue`](Error::CorruptedTaskQueue) error if a failure is planned + /// Return a [`PlannedFailure`](Error::PlannedFailure) error if a failure is planned /// for the given location and current run loop iteration. pub fn maybe_fail(&self, location: FailureLocation) -> Result<()> { if self.planned_failures.contains(&(*self.run_loop_iteration.read().unwrap(), location)) @@ -1435,7 +1429,7 @@ mod tests { FailureLocation::PanicInsideProcessBatch => { panic!("simulated panic") } - _ => Err(Error::CorruptedTaskQueue), + _ => Err(Error::PlannedFailure), } } else { Ok(()) diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap b/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap index 18edc3613..e66ffa567 100644 --- a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap +++ b/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_addition/document_addition_failed.snap @@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Corrupted task queue.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} +0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap b/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap index d2b06a4f4..578b94061 100644 --- a/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Corrupted task queue.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap b/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap index fe3b1eed4..6f59e49a2 100644 --- a/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap +++ b/index-scheduler/src/snapshots/lib.rs/query_tasks_simple/end.snap @@ -8,7 +8,7 @@ source: index-scheduler/src/lib.rs ### All Tasks: 0 {uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: failed, error: ResponseError { code: 200, message: "Corrupted task queue.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +2 {uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued []