From 6d4981ec250fd9e09389681022a4c96ee5497215 Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Fri, 23 Jun 2023 07:24:25 +0800 Subject: [PATCH 1/7] Expose lastUpdate and isIndexing in /stats endpoint --- index-scheduler/src/lib.rs | 24 ++++++++++++++++++++++++ meilisearch/src/metrics.rs | 10 ++++++++++ meilisearch/src/routes/metrics.rs | 3 +++ 3 files changed, 37 insertions(+) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 0f0ecf52a..c1b2deb27 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -794,6 +794,12 @@ impl IndexScheduler { Ok(res) } + // Return true if there is at least one task that is processing. + pub fn is_task_processing(&self) -> Result { + let processing_tasks = self.processing_tasks.read().unwrap().processing.len(); + Ok(processing_tasks > 0) + } + /// Return true iff there is at least one task associated with this index /// that is processing. pub fn is_index_processing(&self, index: &str) -> Result { @@ -1808,6 +1814,24 @@ mod tests { snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_third_task"); } + #[test] + fn test_task_is_processing() { + let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); + + let (file0, documents_count0) = sample_documents(&index_scheduler, 0, 0); + file0.persist().unwrap(); + + let _ = index_scheduler + .register(replace_document_import_task("catto", None, 0, documents_count0)) + .unwrap(); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task"); + + handle.advance_till([Start, BatchCreated, InsideProcessBatch]); + snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_task_processing"); + + assert_eq!(index_scheduler.is_task_processing().unwrap(), true); + } + /// We send a lot of tasks but notify the tasks scheduler only once as /// we send them very fast, we must make sure that they are all processed. #[test] diff --git a/meilisearch/src/metrics.rs b/meilisearch/src/metrics.rs index 79332f360..009c3e472 100644 --- a/meilisearch/src/metrics.rs +++ b/meilisearch/src/metrics.rs @@ -50,4 +50,14 @@ lazy_static! { &["kind", "value"] ) .expect("Can't create a metric"); + pub static ref MEILISEARCH_LAST_UPDATE: IntGauge = register_int_gauge!(opts!( + "meilisearch_last_update", + "Meilisearch Last Update" + )) + .expect("Can't create a metric"); + pub static ref MEILISEARCH_IS_INDEXING: IntGauge = register_int_gauge!(opts!( + "meilisearch_is_indexing", + "Meilisearch Is Indexing" + )) + .expect("Can't create a metric"); } diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index a7d41e33e..f6ee3291b 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -48,6 +48,9 @@ pub async fn get_metrics( } } + crate::metrics::MEILISEARCH_LAST_UPDATE.set(response.last_update.unwrap().unix_timestamp() as i64); + crate::metrics::MEILISEARCH_IS_INDEXING.set(index_scheduler.is_task_processing().unwrap() as i64); + let encoder = TextEncoder::new(); let mut buffer = vec![]; encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics"); From e3fc7112bc3b449ffdcc5d3d62a4ef2ea07fc43e Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Thu, 29 Jun 2023 11:46:47 +0800 Subject: [PATCH 2/7] use `RoaringBitmap::is_empty` instead --- index-scheduler/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index c1b2deb27..fc017395e 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -796,8 +796,7 @@ impl IndexScheduler { // Return true if there is at least one task that is processing. pub fn is_task_processing(&self) -> Result { - let processing_tasks = self.processing_tasks.read().unwrap().processing.len(); - Ok(processing_tasks > 0) + Ok(self.processing_tasks.read().unwrap().processing.is_empty()) } /// Return true iff there is at least one task associated with this index From a5a31667b01f532473d8bd0218c5c805d5266d96 Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Fri, 30 Jun 2023 11:28:18 +0800 Subject: [PATCH 3/7] fix converse result of is_task_processing() --- index-scheduler/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index fc017395e..1f984e733 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -796,7 +796,7 @@ impl IndexScheduler { // Return true if there is at least one task that is processing. pub fn is_task_processing(&self) -> Result { - Ok(self.processing_tasks.read().unwrap().processing.is_empty()) + Ok(!self.processing_tasks.read().unwrap().processing.is_empty()) } /// Return true iff there is at least one task associated with this index From 3bdf01bc1c11dea3b38f2ed3cac78aeb89402272 Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Fri, 30 Jun 2023 17:39:23 +0800 Subject: [PATCH 4/7] Fix failed test --- index-scheduler/src/lib.rs | 10 +--------- meilisearch/src/routes/metrics.rs | 6 ++++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 1f984e733..0594e15e0 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -1817,17 +1817,9 @@ mod tests { fn test_task_is_processing() { let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); - let (file0, documents_count0) = sample_documents(&index_scheduler, 0, 0); - file0.persist().unwrap(); - - let _ = index_scheduler - .register(replace_document_import_task("catto", None, 0, documents_count0)) - .unwrap(); + index_scheduler.register(index_creation_task("index_a", "id")).unwrap(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task"); - handle.advance_till([Start, BatchCreated, InsideProcessBatch]); - snapshot!(snapshot_index_scheduler(&index_scheduler), name: "initial_task_processing"); - assert_eq!(index_scheduler.is_task_processing().unwrap(), true); } diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index f6ee3291b..d35778881 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -48,8 +48,10 @@ pub async fn get_metrics( } } - crate::metrics::MEILISEARCH_LAST_UPDATE.set(response.last_update.unwrap().unix_timestamp() as i64); - crate::metrics::MEILISEARCH_IS_INDEXING.set(index_scheduler.is_task_processing().unwrap() as i64); + crate::metrics::MEILISEARCH_LAST_UPDATE + .set(response.last_update.unwrap().unix_timestamp()); + crate::metrics::MEILISEARCH_IS_INDEXING + .set(index_scheduler.is_task_processing().unwrap() as i64); let encoder = TextEncoder::new(); let mut buffer = vec![]; From 9859e65d2fa229a5a95862630cfcbf3610b5ac39 Mon Sep 17 00:00:00 2001 From: Cong Chen Date: Sat, 1 Jul 2023 09:32:50 +0800 Subject: [PATCH 5/7] fix tests --- index-scheduler/src/lib.rs | 4 ++-- meilisearch/src/routes/metrics.rs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 0594e15e0..a9affed0b 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -1815,12 +1815,12 @@ mod tests { #[test] fn test_task_is_processing() { - let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); + let (index_scheduler, _handle) = IndexScheduler::test(true, vec![]); index_scheduler.register(index_creation_task("index_a", "id")).unwrap(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task"); - assert_eq!(index_scheduler.is_task_processing().unwrap(), true); + assert!(index_scheduler.is_task_processing().unwrap()); } /// We send a lot of tasks but notify the tasks scheduler only once as diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index d35778881..534c0c500 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -48,8 +48,7 @@ pub async fn get_metrics( } } - crate::metrics::MEILISEARCH_LAST_UPDATE - .set(response.last_update.unwrap().unix_timestamp()); + crate::metrics::MEILISEARCH_LAST_UPDATE.set(response.last_update.unwrap().unix_timestamp()); crate::metrics::MEILISEARCH_IS_INDEXING .set(index_scheduler.is_task_processing().unwrap() as i64); From 71500a4e150f80be90ada52e1e7aa316c626cc93 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Mon, 3 Jul 2023 11:20:43 +0200 Subject: [PATCH 6/7] Update tests --- index-scheduler/src/lib.rs | 3 +- .../registered_a_task.snap | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index a9affed0b..5bab13a2d 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -1815,11 +1815,12 @@ mod tests { #[test] fn test_task_is_processing() { - let (index_scheduler, _handle) = IndexScheduler::test(true, vec![]); + let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]); index_scheduler.register(index_creation_task("index_a", "id")).unwrap(); snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_a_task"); + handle.advance_till([Start, BatchCreated]); assert!(index_scheduler.is_task_processing().unwrap()); } diff --git a/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap b/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap new file mode 100644 index 000000000..f17bfe38f --- /dev/null +++ b/index-scheduler/src/snapshots/lib.rs/test_task_is_processing/registered_a_task.snap @@ -0,0 +1,36 @@ +--- +source: index-scheduler/src/lib.rs +--- +### Autobatching Enabled = true +### Processing Tasks: +[] +---------------------------------------------------------------------- +### All Tasks: +0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +---------------------------------------------------------------------- +### Status: +enqueued [0,] +---------------------------------------------------------------------- +### Kind: +"indexCreation" [0,] +---------------------------------------------------------------------- +### Index Tasks: +index_a [0,] +---------------------------------------------------------------------- +### Index Mapper: + +---------------------------------------------------------------------- +### Canceled By: + +---------------------------------------------------------------------- +### Enqueued At: +[timestamp] [0,] +---------------------------------------------------------------------- +### Started At: +---------------------------------------------------------------------- +### Finished At: +---------------------------------------------------------------------- +### File Store: + +---------------------------------------------------------------------- + From 5387cf17186dfd977275a5e30fc406d96f1e698b Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 3 Jul 2023 15:21:58 +0200 Subject: [PATCH 7/7] Don't unwrap in case of error/missing last_update field --- meilisearch/src/metrics.rs | 16 ++++++---------- meilisearch/src/routes/metrics.rs | 7 ++++--- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/meilisearch/src/metrics.rs b/meilisearch/src/metrics.rs index 009c3e472..2becdae76 100644 --- a/meilisearch/src/metrics.rs +++ b/meilisearch/src/metrics.rs @@ -50,14 +50,10 @@ lazy_static! { &["kind", "value"] ) .expect("Can't create a metric"); - pub static ref MEILISEARCH_LAST_UPDATE: IntGauge = register_int_gauge!(opts!( - "meilisearch_last_update", - "Meilisearch Last Update" - )) - .expect("Can't create a metric"); - pub static ref MEILISEARCH_IS_INDEXING: IntGauge = register_int_gauge!(opts!( - "meilisearch_is_indexing", - "Meilisearch Is Indexing" - )) - .expect("Can't create a metric"); + pub static ref MEILISEARCH_LAST_UPDATE: IntGauge = + register_int_gauge!(opts!("meilisearch_last_update", "Meilisearch Last Update")) + .expect("Can't create a metric"); + pub static ref MEILISEARCH_IS_INDEXING: IntGauge = + register_int_gauge!(opts!("meilisearch_is_indexing", "Meilisearch Is Indexing")) + .expect("Can't create a metric"); } diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index 534c0c500..93d07e67c 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -48,9 +48,10 @@ pub async fn get_metrics( } } - crate::metrics::MEILISEARCH_LAST_UPDATE.set(response.last_update.unwrap().unix_timestamp()); - crate::metrics::MEILISEARCH_IS_INDEXING - .set(index_scheduler.is_task_processing().unwrap() as i64); + if let Some(last_update) = response.last_update { + crate::metrics::MEILISEARCH_LAST_UPDATE.set(last_update.unix_timestamp()); + } + crate::metrics::MEILISEARCH_IS_INDEXING.set(index_scheduler.is_task_processing()? as i64); let encoder = TextEncoder::new(); let mut buffer = vec![];