From 84f701679d0f1403d8e6f8df45925248719a4171 Mon Sep 17 00:00:00 2001
From: Louis Dureuil <louis@meilisearch.com>
Date: Wed, 15 Nov 2023 23:06:19 +0100
Subject: [PATCH] Get panics in index-scheduler

---
 index-scheduler/src/error.rs          | 13 +++++++++----
 index-scheduler/src/insta_snapshot.rs |  1 +
 index-scheduler/src/lib.rs            | 16 +++++++++++++++-
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/index-scheduler/src/error.rs b/index-scheduler/src/error.rs
index bbe526460..eff3a5089 100644
--- a/index-scheduler/src/error.rs
+++ b/index-scheduler/src/error.rs
@@ -117,8 +117,13 @@ pub enum Error {
     Heed(#[from] heed::Error),
     #[error(transparent)]
     Milli(#[from] milli::Error),
-    #[error("An unexpected crash occurred when processing the task.")]
-    ProcessBatchPanicked,
+    #[error("An unexpected crash occurred when processing the task. {}", {
+        match .0 {
+            Some(report) => format!("Get /reports/{}", report),
+            None => "No report was saved.".into(),
+        }
+    })]
+    ProcessBatchPanicked(Option<uuid::Uuid>),
     #[error(transparent)]
     FileStore(#[from] file_store::Error),
     #[error(transparent)]
@@ -181,7 +186,7 @@ impl Error {
             | Error::Dump(_)
             | Error::Heed(_)
             | Error::Milli(_)
-            | Error::ProcessBatchPanicked
+            | Error::ProcessBatchPanicked(_)
             | Error::FileStore(_)
             | Error::IoError(_)
             | Error::Persist(_)
@@ -224,7 +229,7 @@ impl ErrorCode for Error {
             Error::NoSpaceLeftInTaskQueue => Code::NoSpaceLeftOnDevice,
             Error::Dump(e) => e.error_code(),
             Error::Milli(e) => e.error_code(),
-            Error::ProcessBatchPanicked => Code::Internal,
+            Error::ProcessBatchPanicked(_) => Code::Internal,
             Error::Heed(e) => e.error_code(),
             Error::HeedTransaction(e) => e.error_code(),
             Error::FileStore(e) => e.error_code(),
diff --git a/index-scheduler/src/insta_snapshot.rs b/index-scheduler/src/insta_snapshot.rs
index f820ce99d..e484b4f54 100644
--- a/index-scheduler/src/insta_snapshot.rs
+++ b/index-scheduler/src/insta_snapshot.rs
@@ -39,6 +39,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         test_breakpoint_sdr: _,
         planned_failures: _,
         run_loop_iteration: _,
+        panic_reader: _,
     } = scheduler;
 
     let rtxn = env.read_txn().unwrap();
diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs
index 03c5c3b0e..fe50d21d6 100644
--- a/index-scheduler/src/lib.rs
+++ b/index-scheduler/src/lib.rs
@@ -54,6 +54,7 @@ use meilisearch_types::milli::documents::DocumentsBatchBuilder;
 use meilisearch_types::milli::update::IndexerConfig;
 use meilisearch_types::milli::{self, CboRoaringBitmapCodec, Index, RoaringBitmapCodec, BEU32};
 use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
+use panic_hook::PanicReader;
 use puffin::FrameView;
 use roaring::RoaringBitmap;
 use synchronoise::SignalEvent;
@@ -332,6 +333,8 @@ pub struct IndexScheduler {
     /// The path to the version file of Meilisearch.
     pub(crate) version_file_path: PathBuf,
 
+    pub(crate) panic_reader: PanicReader,
+
     // ================= test
     // The next entry is dedicated to the tests.
     /// Provide a way to set a breakpoint in multiple part of the scheduler.
@@ -382,6 +385,7 @@ impl IndexScheduler {
             #[cfg(test)]
             run_loop_iteration: self.run_loop_iteration.clone(),
             features: self.features.clone(),
+            panic_reader: self.panic_reader.clone(),
         }
     }
 }
@@ -439,6 +443,12 @@ impl IndexScheduler {
         let finished_at = env.create_database(&mut wtxn, Some(db_name::FINISHED_AT))?;
         wtxn.commit()?;
 
+        const MAX_REPORT_COUNT: usize = 20;
+
+        let panic_reader = panic_hook::PanicReader::install_panic_hook(
+            std::num::NonZeroUsize::new(MAX_REPORT_COUNT).unwrap(),
+        );
+
         // allow unreachable_code to get rids of the warning in the case of a test build.
         let this = Self {
             must_stop_processing: MustStopProcessing::default(),
@@ -479,6 +489,7 @@ impl IndexScheduler {
             #[cfg(test)]
             run_loop_iteration: Arc::new(RwLock::new(0)),
             features,
+            panic_reader,
         };
 
         this.run();
@@ -1131,7 +1142,10 @@ impl IndexScheduler {
                 .name(String::from("batch-operation"))
                 .spawn(move || cloned_index_scheduler.process_batch(batch))
                 .unwrap();
-            handle.join().unwrap_or(Err(Error::ProcessBatchPanicked))
+
+            self.panic_reader
+                .join_thread(handle)
+                .unwrap_or_else(|maybe_report| Err(Error::ProcessBatchPanicked(maybe_report)))
         };
 
         #[cfg(test)]