add a view for the task queue in the metrics

This commit is contained in:
Tamo 2023-05-25 12:58:13 +02:00
parent 1b601f70c6
commit c433bdd1cd
4 changed files with 1352 additions and 962 deletions

View File

@ -24,7 +24,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1,
"id": 4,
"links": [],
"liveNow": false,
"panels": [
@ -44,8 +44,7 @@
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "i51CxikVz"
"type": "prometheus"
},
"refId": "A"
}
@ -55,8 +54,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -103,12 +101,11 @@
"showThresholdMarkers": true,
"text": {}
},
"pluginVersion": "9.0.5",
"pluginVersion": "9.5.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -124,8 +121,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -169,12 +165,11 @@
"showThresholdMarkers": true,
"text": {}
},
"pluginVersion": "9.0.5",
"pluginVersion": "9.5.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -190,8 +185,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -231,12 +225,11 @@
"showThresholdMarkers": true,
"text": {}
},
"pluginVersion": "9.0.5",
"pluginVersion": "9.5.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"expr": "meilisearch_index_docs_count{job=\"meilisearch\", index=\"$Index\", instance=\"$instance\"}",
@ -250,8 +243,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -295,12 +287,11 @@
"showThresholdMarkers": true,
"text": {}
},
"pluginVersion": "9.0.5",
"pluginVersion": "9.5.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -316,8 +307,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -357,12 +347,11 @@
"showThresholdMarkers": true,
"text": {}
},
"pluginVersion": "9.0.5",
"pluginVersion": "9.5.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -378,8 +367,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -419,12 +407,11 @@
"showThresholdMarkers": true,
"text": {}
},
"pluginVersion": "9.0.5",
"pluginVersion": "9.5.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -440,8 +427,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -449,6 +435,8 @@
"mode": "continuous-YlBl"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
@ -506,7 +494,8 @@
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right"
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
@ -516,8 +505,7 @@
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -533,8 +521,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -542,6 +529,8 @@
"mode": "continuous-YlBl"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
@ -599,7 +588,8 @@
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right"
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
@ -610,8 +600,7 @@
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -636,8 +625,22 @@
},
"dataFormat": "tsbuckets",
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 12,
@ -652,13 +655,50 @@
"legend": {
"show": false
},
"pluginVersion": "8.1.4",
"options": {
"calculate": false,
"calculation": {},
"cellGap": 2,
"cellValues": {},
"color": {
"exponent": 0.5,
"fill": "#b4ff00",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Blues",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": false
},
"rowsFrame": {
"layout": "auto"
},
"showValue": "never",
"tooltip": {
"show": true,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"decimals": 2,
"reverse": false,
"unit": "s"
}
},
"pluginVersion": "9.5.2",
"reverseYBuckets": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -689,6 +729,319 @@
"yBucketBound": "auto",
"yBucketNumber": 10
},
{
"datasource": {
"type": "prometheus",
"uid": "bb3298a4-9acf-4da1-b86a-813f29f50888"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "none"
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"succeeded "
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 30
},
"id": 23,
"interval": "5s",
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.1.4",
"targets": [
{
"datasource": {
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
"expr": "nb_tasks{instance=\"$instance\", job=\"meilisearch\", kind=\"statuses\"}",
"interval": "",
"legendFormat": "{{value}} ",
"range": true,
"refId": "A"
}
],
"title": "Number of tasks by statuses",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "bb3298a4-9acf-4da1-b86a-813f29f50888"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 12,
"x": 12,
"y": 30
},
"id": 24,
"interval": "5s",
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.1.4",
"targets": [
{
"datasource": {
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
"expr": "nb_tasks{instance=\"$instance\", job=\"meilisearch\", kind=\"types\"}",
"interval": "",
"legendFormat": "{{value}} ",
"range": true,
"refId": "A"
}
],
"title": "Number of tasks by types",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "bb3298a4-9acf-4da1-b86a-813f29f50888"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 41
},
"id": 25,
"interval": "5s",
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "8.1.4",
"targets": [
{
"datasource": {
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
"expr": "nb_tasks{instance=\"$instance\", job=\"meilisearch\", kind=\"indexes\"}",
"interval": "",
"legendFormat": "{{value}} ",
"range": true,
"refId": "A"
}
],
"title": "Number of tasks by indexes",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": {
@ -699,7 +1052,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 30
"y": 52
},
"id": 12,
"panels": [],
@ -717,8 +1070,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -726,6 +1078,8 @@
"mode": "continuous-YlBl"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
@ -776,7 +1130,7 @@
"h": 11,
"w": 12,
"x": 0,
"y": 31
"y": 53
},
"id": 4,
"interval": "5s",
@ -784,7 +1138,8 @@
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
@ -794,8 +1149,7 @@
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -807,8 +1161,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"exemplar": true,
"expr": "sum(rate(container_cpu_usage_seconds_total{name='mongodb-redis'}[1m])) by (name)",
@ -822,8 +1175,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
@ -831,6 +1183,8 @@
"mode": "continuous-YlBl"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "MiB",
"axisPlacement": "left",
"barAlignment": 0,
@ -879,7 +1233,7 @@
"h": 11,
"w": 12,
"x": 12,
"y": 31
"y": 53
},
"id": 5,
"interval": "5s",
@ -887,7 +1241,8 @@
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
@ -897,8 +1252,7 @@
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"editorMode": "builder",
"exemplar": true,
@ -910,8 +1264,7 @@
},
{
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"type": "prometheus"
},
"exemplar": true,
"expr": "container_memory_usage_bytes{name=\"mongodb-redis\"} / 1024 / 1024",
@ -925,7 +1278,7 @@
}
],
"refresh": "5s",
"schemaVersion": 36,
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
@ -938,7 +1291,7 @@
},
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"uid": "bb3298a4-9acf-4da1-b86a-813f29f50888"
},
"definition": "label_values(instance)",
"hide": 0,
@ -960,12 +1313,12 @@
{
"current": {
"selected": false,
"text": "movie-collection",
"value": "movie-collection"
"text": "mieli",
"value": "mieli"
},
"datasource": {
"type": "prometheus",
"uid": "1MRsknzVz"
"uid": "bb3298a4-9acf-4da1-b86a-813f29f50888"
},
"definition": "label_values(index)",
"hide": 0,
@ -987,7 +1340,7 @@
]
},
"time": {
"from": "now-15m",
"from": "now-5m",
"to": "now"
},
"timepicker": {
@ -1002,6 +1355,6 @@
"timezone": "",
"title": "Meilisearch",
"uid": "7wcZ94dnz",
"version": 47,
"version": 6,
"weekStart": ""
}

View File

@ -31,7 +31,7 @@ mod uuid_codec;
pub type Result<T> = std::result::Result<T, Error>;
pub type TaskId = u32;
use std::collections::HashMap;
use std::collections::{BTreeMap, HashMap};
use std::ops::{Bound, RangeBounds};
use std::path::{Path, PathBuf};
use std::sync::atomic::AtomicBool;
@ -737,6 +737,34 @@ impl IndexScheduler {
Ok(tasks)
}
pub fn get_stats(&self) -> Result<BTreeMap<String, BTreeMap<String, u64>>> {
let rtxn = self.read_txn()?;
let mut res = BTreeMap::new();
res.insert(
"statuses".to_string(),
enum_iterator::all::<Status>()
.map(|s| Ok((s.to_string(), self.get_status(&rtxn, s)?.len())))
.collect::<Result<BTreeMap<String, u64>>>()?,
);
res.insert(
"types".to_string(),
enum_iterator::all::<Kind>()
.map(|s| Ok((s.to_string(), self.get_kind(&rtxn, s)?.len())))
.collect::<Result<BTreeMap<String, u64>>>()?,
);
res.insert(
"indexes".to_string(),
self.index_tasks
.iter(&rtxn)?
.map(|res| Ok(res.map(|(name, bitmap)| (name.to_string(), bitmap.len()))?))
.collect::<Result<BTreeMap<String, u64>>>()?,
);
Ok(res)
}
/// Return true iff there is at least one task associated with this index
/// that is processing.
pub fn is_index_processing(&self, index: &str) -> Result<bool> {

View File

@ -40,4 +40,7 @@ lazy_static! {
HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec()
)
.expect("Can't create a metric");
pub static ref NB_TASKS: IntGaugeVec =
register_int_gauge_vec!(opts!("nb_tasks", "Number of tasks"), &["kind", "value"])
.expect("Can't create a metric");
}

View File

@ -39,6 +39,12 @@ pub async fn get_metrics(
.set(value.number_of_documents as i64);
}
for (kind, value) in index_scheduler.get_stats()? {
for (value, count) in value {
crate::metrics::NB_TASKS.with_label_values(&[&kind, &value]).set(count as i64);
}
}
let encoder = TextEncoder::new();
let mut buffer = vec![];
encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics");