From de58ccd4ba12f5a977c8c221b77f29c06266d4ba Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Thu, 4 Aug 2022 20:16:09 +0530 Subject: [PATCH 01/20] prometheus and grafana dashboards implemented --- Cargo.lock | 38 ++ grafana-dashboards/dashboard.json | 1007 ++++++++++++++++++++++++++++ meilisearch-http/Cargo.toml | 2 + meilisearch-http/src/lib.rs | 33 + meilisearch-http/src/metrics.rs | 40 ++ meilisearch-http/src/routes/mod.rs | 36 + 6 files changed, 1156 insertions(+) create mode 100644 grafana-dashboards/dashboard.json create mode 100644 meilisearch-http/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 627c9c0b2..647545d8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2045,6 +2045,7 @@ dependencies = [ "indexmap", "itertools", "jsonwebtoken", + "lazy_static", "log", "manifest-dir-macros", "maplit", @@ -2058,6 +2059,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "platform-dirs", + "prometheus", "rand", "rayon", "regex", @@ -2664,6 +2666,36 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0941606b9934e2d98a3677759a971756eb821f75764d0e0d26946d08e74d9104" +dependencies = [ + "bitflags", + "byteorder", + "hex", + "lazy_static", + "libc", +] + +[[package]] +name = "prometheus" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cface98dfa6d645ea4c789839f176e4b072265d085bfcc48eaa8d137f58d3c39" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs", + "protobuf", + "thiserror", +] + [[package]] name = "proptest" version = "1.0.0" @@ -2695,6 +2727,12 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "protobuf" +version = "2.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" + [[package]] name = "quick-error" version = "1.2.3" diff --git a/grafana-dashboards/dashboard.json b/grafana-dashboards/dashboard.json new file mode 100644 index 000000000..fe64e2966 --- /dev/null +++ b/grafana-dashboards/dashboard.json @@ -0,0 +1,1007 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "i51CxikVz" + }, + "refId": "A" + } + ], + "title": "Web application metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 2, + "interval": "5s", + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_database_size{job=\"meilisearch\", instance=\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Database Size", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 22, + "interval": "5s", + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_total_index{job=\"meilisearch\", instance=\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Indexes Count", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 18, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "expr": "meilisearch_docs_count{job=\"meilisearch\", index=\"$Index\", instance=\"$instance\"}", + "hide": false, + "range": true, + "refId": "A" + } + ], + "title": "Total Documents", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 19, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "round(increase(http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"meilisearch\"}[1h]))", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total Searches (1h)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 20, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "round(increase(http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"meilisearch\"}[24h]))", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total Searches (24h)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 21, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "round(increase(http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"meilisearch\"}[30d]))", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total Searches (30d)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 1, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "rate(http_requests_total{instance=\"$instance\", job=\"meilisearch\"}[5m])", + "interval": "", + "legendFormat": "{{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP requests per second (All Indexes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "rate(http_response_time_seconds_sum{instance=\"$instance\", job=\"meilisearch\"}[5m]) / rate(http_response_time_seconds_count[5m])", + "interval": "", + "legendFormat": "{{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Mean response time (All Indexes)", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateBlues", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 18 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 16, + "legend": { + "show": false + }, + "pluginVersion": "8.1.4", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum by(le) (increase(http_response_time_seconds_bucket{path=\"/indexes/$Index/search\", instance=\"$instance\", job=\"meilisearch\"}[30s]))", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Response time distribution over time (`POST /indexes/:index/search`)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": 10, + "yAxis": { + "decimals": 2, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto", + "yBucketNumber": 10 + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "i51CxikVz" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "i51CxikVz" + }, + "refId": "A" + } + ], + "title": "System metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 4, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "rate(process_cpu_seconds_total{job=\"meilisearch\", instance=\"$instance\"}[1m])", + "interval": "", + "legendFormat": "process", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{name='mongodb-redis'}[1m])) by (name)", + "interval": "", + "legendFormat": "container", + "refId": "B" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "MiB", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 5, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "process_resident_memory_bytes{job=\"meilisearch\", instance=\"$instance\"} / 1024 / 1024", + "interval": "", + "legendFormat": "process", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "exemplar": true, + "expr": "container_memory_usage_bytes{name=\"mongodb-redis\"} / 1024 / 1024", + "interval": "", + "legendFormat": "container", + "refId": "B" + } + ], + "title": "Memory usage", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "localhost:7700", + "value": "localhost:7700" + }, + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "definition": "label_values(instance)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "movie-collection", + "value": "movie-collection" + }, + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "definition": "label_values(index)", + "hide": 0, + "includeAll": false, + "label": "index", + "multi": false, + "name": "Index", + "options": [], + "query": { + "query": "label_values(index)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "", + "title": "MeiliSearch", + "uid": "7wcZ94dnz", + "version": 47, + "weekStart": "" + } \ No newline at end of file diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 81e0205da..9395a604c 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -77,6 +77,8 @@ tokio = { version = "1.17.0", features = ["full"] } tokio-stream = "0.1.8" uuid = { version = "1.1.2", features = ["serde", "v4"] } walkdir = "2.3.2" +prometheus = { version = "0.13.0", features = ["process"] } +lazy_static = "1.4.0" [dev-dependencies] actix-rt = "2.7.0" diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index fcf07587f..a4395f606 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -7,6 +7,7 @@ pub mod task; pub mod extractors; pub mod option; pub mod routes; +pub mod metrics; use std::sync::{atomic::AtomicBool, Arc}; use std::time::Duration; @@ -146,16 +147,48 @@ macro_rules! create_app { use actix_cors::Cors; use actix_web::middleware::TrailingSlash; use actix_web::App; + use actix_web::dev::Service; use actix_web::{middleware, web}; use meilisearch_http::error::MeilisearchHttpError; use meilisearch_http::routes; use meilisearch_http::{configure_data, dashboard}; use meilisearch_types::error::ResponseError; + use meilisearch_http::metrics; + + use lazy_static::lazy_static; + use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge}; + use prometheus::{HistogramVec, IntCounterVec, IntGauge, HistogramTimer}; App::new() .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics)) .configure(routes::configure) .configure(|s| dashboard(s, $enable_frontend)) + .wrap_fn(|req, srv| { + let mut histogram_timer: Option = None; + let request_path = req.path(); + let is_registered_resource = req.resource_map().has_resource(request_path); + if is_registered_resource { + let request_method = req.method().to_string(); + histogram_timer = Some( + metrics::HTTP_RESPONSE_TIME_SECONDS + .with_label_values(&[&request_method, request_path]) + .start_timer(), + ); + metrics::HTTP_REQUESTS_TOTAL + .with_label_values(&[&request_method, request_path]) + .inc(); + } + + let fut = srv.call(req); + + async { + let res = fut.await?; + if let Some(histogram_timer) = histogram_timer { + histogram_timer.observe_duration(); + }; + Ok(res) + } + }) .wrap( Cors::default() .send_wildcard() diff --git a/meilisearch-http/src/metrics.rs b/meilisearch-http/src/metrics.rs new file mode 100644 index 000000000..b93696dd3 --- /dev/null +++ b/meilisearch-http/src/metrics.rs @@ -0,0 +1,40 @@ +use lazy_static::lazy_static; +use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, register_int_gauge_vec}; +use prometheus::{HistogramVec, IntCounterVec, IntGauge, IntGaugeVec}; + +const HTTP_RESPONSE_TIME_CUSTOM_BUCKETS: &[f64; 14] = &[ + 0.0005, 0.0008, 0.00085, 0.0009, 0.00095, 0.001, 0.00105, 0.0011, 0.00115, 0.0012, 0.0015, + 0.002, 0.003, 1.0, +]; + +lazy_static! { + pub static ref HTTP_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!( + opts!("http_requests_total", "HTTP requests total"), + &["method", "path"] + ) + .expect("Can't create a metric"); + + pub static ref MEILISEARCH_DB_SIZE: IntGauge = register_int_gauge!( + opts!("meilisearch_database_size", "MeiliSearch Stats DbSize") + ) + .expect("Can't create a metric"); + + pub static ref MEILISEARCH_INDEX_COUNT: IntGauge = register_int_gauge!( + opts!("meilisearch_total_index", "MeiliSearch Stats Index Count") + ) + .expect("Can't create a metric"); + + pub static ref MEILISEARCH_DOCS_COUNT: IntGaugeVec = register_int_gauge_vec!( + opts!("meilisearch_docs_count", "MeiliSearch Stats Docs Count"), + &["index"] + ) + .expect("Can't create a metric"); + + pub static ref HTTP_RESPONSE_TIME_SECONDS: HistogramVec = register_histogram_vec!( + "http_response_time_seconds", + "HTTP response times", + &["method", "path"], + HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec() + ) + .expect("Can't create a metric"); +} \ No newline at end of file diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index f61854c48..987e50aca 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,4 +1,5 @@ use actix_web::{web, HttpResponse}; +use actix_web::http::header::{self}; use log::debug; use serde::{Deserialize, Serialize}; @@ -10,6 +11,7 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::star_or::StarOr; use crate::extractors::authentication::{policies::*, GuardedData}; +use prometheus::{Encoder, TextEncoder}; mod api_key; mod dump; @@ -19,6 +21,7 @@ mod tasks; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::scope("/tasks").configure(tasks::configure)) .service(web::resource("/health").route(web::get().to(get_health))) + .service(web::resource("/metrics").route(web::get().to(get_metrics))) .service(web::scope("/keys").configure(api_key::configure)) .service(web::scope("/dumps").configure(dump::configure)) .service(web::resource("/stats").route(web::get().to(get_stats))) @@ -269,3 +272,36 @@ struct KeysResponse { pub async fn get_health() -> Result { Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) } + +pub async fn get_metrics( + meilisearch: GuardedData, MeiliSearch>, +) -> Result { + + let search_rules = &meilisearch.filters().search_rules; + let response = meilisearch.get_all_stats(search_rules).await?; + + crate::metrics::MEILISEARCH_DB_SIZE + .set(response.database_size as i64); + + crate::metrics::MEILISEARCH_INDEX_COUNT + .set(response.indexes.len() as i64); + + for (index, value) in response.indexes.iter() { + crate::metrics::MEILISEARCH_DOCS_COUNT + .with_label_values(&[&index]) + .set(value.number_of_documents as i64); + } + + let encoder = TextEncoder::new(); + let mut buffer = vec![]; + encoder + .encode(&prometheus::gather(), &mut buffer) + .expect("Failed to encode metrics"); + + let response = String::from_utf8(buffer.clone()).expect("Failed to convert bytes to string"); + buffer.clear(); + + Ok(HttpResponse::Ok() + .insert_header(header::ContentType(mime::TEXT_PLAIN)) + .body(response)) +} From d08c77706ca5813a1515b4dad020dec07a56350a Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Mon, 8 Aug 2022 11:11:38 +0530 Subject: [PATCH 02/20] clippy & rustfmt fixed --- meilisearch-http/src/lib.rs | 11 ++++++----- meilisearch-http/src/metrics.rs | 25 +++++++++++++------------ meilisearch-http/src/routes/mod.rs | 13 +++++-------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index a4395f606..f0ce0396c 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -7,7 +7,6 @@ pub mod task; pub mod extractors; pub mod option; pub mod routes; -pub mod metrics; use std::sync::{atomic::AtomicBool, Arc}; use std::time::Duration; @@ -145,19 +144,21 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { macro_rules! create_app { ($data:expr, $auth:expr, $enable_frontend:expr, $opt:expr, $analytics:expr) => {{ use actix_cors::Cors; + use actix_web::dev::Service; use actix_web::middleware::TrailingSlash; use actix_web::App; - use actix_web::dev::Service; use actix_web::{middleware, web}; use meilisearch_http::error::MeilisearchHttpError; + use meilisearch_http::metrics; use meilisearch_http::routes; use meilisearch_http::{configure_data, dashboard}; use meilisearch_types::error::ResponseError; - use meilisearch_http::metrics; use lazy_static::lazy_static; - use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge}; - use prometheus::{HistogramVec, IntCounterVec, IntGauge, HistogramTimer}; + use prometheus::{ + opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, + }; + use prometheus::{HistogramTimer, HistogramVec, IntCounterVec, IntGauge}; App::new() .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics)) diff --git a/meilisearch-http/src/metrics.rs b/meilisearch-http/src/metrics.rs index b93696dd3..8786d7aeb 100644 --- a/meilisearch-http/src/metrics.rs +++ b/meilisearch-http/src/metrics.rs @@ -1,5 +1,8 @@ use lazy_static::lazy_static; -use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, register_int_gauge_vec}; +use prometheus::{ + opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, + register_int_gauge_vec, +}; use prometheus::{HistogramVec, IntCounterVec, IntGauge, IntGaugeVec}; const HTTP_RESPONSE_TIME_CUSTOM_BUCKETS: &[f64; 14] = &[ @@ -13,23 +16,21 @@ lazy_static! { &["method", "path"] ) .expect("Can't create a metric"); - - pub static ref MEILISEARCH_DB_SIZE: IntGauge = register_int_gauge!( - opts!("meilisearch_database_size", "MeiliSearch Stats DbSize") - ) + pub static ref MEILISEARCH_DB_SIZE: IntGauge = register_int_gauge!(opts!( + "meilisearch_database_size", + "MeiliSearch Stats DbSize" + )) .expect("Can't create a metric"); - - pub static ref MEILISEARCH_INDEX_COUNT: IntGauge = register_int_gauge!( - opts!("meilisearch_total_index", "MeiliSearch Stats Index Count") - ) + pub static ref MEILISEARCH_INDEX_COUNT: IntGauge = register_int_gauge!(opts!( + "meilisearch_total_index", + "MeiliSearch Stats Index Count" + )) .expect("Can't create a metric"); - pub static ref MEILISEARCH_DOCS_COUNT: IntGaugeVec = register_int_gauge_vec!( opts!("meilisearch_docs_count", "MeiliSearch Stats Docs Count"), &["index"] ) .expect("Can't create a metric"); - pub static ref HTTP_RESPONSE_TIME_SECONDS: HistogramVec = register_histogram_vec!( "http_response_time_seconds", "HTTP response times", @@ -37,4 +38,4 @@ lazy_static! { HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec() ) .expect("Can't create a metric"); -} \ No newline at end of file +} diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 987e50aca..469e747d8 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,5 +1,5 @@ -use actix_web::{web, HttpResponse}; use actix_web::http::header::{self}; +use actix_web::{web, HttpResponse}; use log::debug; use serde::{Deserialize, Serialize}; @@ -276,20 +276,17 @@ pub async fn get_health() -> Result { pub async fn get_metrics( meilisearch: GuardedData, MeiliSearch>, ) -> Result { - let search_rules = &meilisearch.filters().search_rules; let response = meilisearch.get_all_stats(search_rules).await?; - crate::metrics::MEILISEARCH_DB_SIZE - .set(response.database_size as i64); + crate::metrics::MEILISEARCH_DB_SIZE.set(response.database_size as i64); - crate::metrics::MEILISEARCH_INDEX_COUNT - .set(response.indexes.len() as i64); + crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64); for (index, value) in response.indexes.iter() { crate::metrics::MEILISEARCH_DOCS_COUNT - .with_label_values(&[&index]) - .set(value.number_of_documents as i64); + .with_label_values(&[index]) + .set(value.number_of_documents as i64); } let encoder = TextEncoder::new(); From 93daa4c464faca3a5a4841d9571fe86e710b700b Mon Sep 17 00:00:00 2001 From: Pavo-Tusker <43550760+pavo-tusker@users.noreply.github.com> Date: Thu, 11 Aug 2022 11:38:06 +0530 Subject: [PATCH 03/20] Update meilisearch-http/src/routes/mod.rs Co-authored-by: Tamo --- meilisearch-http/src/routes/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 469e747d8..6d7a6f27a 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -280,7 +280,6 @@ pub async fn get_metrics( let response = meilisearch.get_all_stats(search_rules).await?; crate::metrics::MEILISEARCH_DB_SIZE.set(response.database_size as i64); - crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64); for (index, value) in response.indexes.iter() { From 225d9936ed35139847b66a1905d7c82d134ec8da Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Fri, 12 Aug 2022 12:44:14 +0530 Subject: [PATCH 04/20] review fixes --- meilisearch-http/src/routes/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 6d7a6f27a..8f5c9f3d4 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -294,8 +294,7 @@ pub async fn get_metrics( .encode(&prometheus::gather(), &mut buffer) .expect("Failed to encode metrics"); - let response = String::from_utf8(buffer.clone()).expect("Failed to convert bytes to string"); - buffer.clear(); + let response = String::from_utf8(buffer).expect("Failed to convert bytes to string"); Ok(HttpResponse::Ok() .insert_header(header::ContentType(mime::TEXT_PLAIN)) From cd1845948468f68a537d84ae3a33da01e5cafa69 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Wed, 17 Aug 2022 17:14:55 +0530 Subject: [PATCH 05/20] Changed prometheus metrics feature as optional --- meilisearch-http/src/lib.rs | 48 ++++++----------- meilisearch-http/src/option.rs | 6 ++- meilisearch-http/src/route_metrics.rs | 75 +++++++++++++++++++++++++++ meilisearch-http/src/routes/mod.rs | 1 - 4 files changed, 95 insertions(+), 35 deletions(-) create mode 100644 meilisearch-http/src/route_metrics.rs diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index f0ce0396c..edffb6afc 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -6,6 +6,7 @@ pub mod task; #[macro_use] pub mod extractors; pub mod option; +pub mod route_metrics; pub mod routes; use std::sync::{atomic::AtomicBool, Arc}; @@ -140,56 +141,33 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { config.service(web::resource("/").route(web::get().to(routes::running))); } +pub fn configure_metrics_route(config: &mut web::ServiceConfig, enable_metrics_route: bool) { + if enable_metrics_route { + config.service(web::resource("/metrics").route(web::get().to(routes::get_metrics))); + } +} + #[macro_export] macro_rules! create_app { ($data:expr, $auth:expr, $enable_frontend:expr, $opt:expr, $analytics:expr) => {{ use actix_cors::Cors; use actix_web::dev::Service; + use actix_web::middleware::Condition; use actix_web::middleware::TrailingSlash; use actix_web::App; use actix_web::{middleware, web}; use meilisearch_http::error::MeilisearchHttpError; use meilisearch_http::metrics; + use meilisearch_http::route_metrics; use meilisearch_http::routes; - use meilisearch_http::{configure_data, dashboard}; + use meilisearch_http::{configure_data, configure_metrics_route, dashboard}; use meilisearch_types::error::ResponseError; - use lazy_static::lazy_static; - use prometheus::{ - opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, - }; - use prometheus::{HistogramTimer, HistogramVec, IntCounterVec, IntGauge}; - App::new() .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics)) .configure(routes::configure) .configure(|s| dashboard(s, $enable_frontend)) - .wrap_fn(|req, srv| { - let mut histogram_timer: Option = None; - let request_path = req.path(); - let is_registered_resource = req.resource_map().has_resource(request_path); - if is_registered_resource { - let request_method = req.method().to_string(); - histogram_timer = Some( - metrics::HTTP_RESPONSE_TIME_SECONDS - .with_label_values(&[&request_method, request_path]) - .start_timer(), - ); - metrics::HTTP_REQUESTS_TOTAL - .with_label_values(&[&request_method, request_path]) - .inc(); - } - - let fut = srv.call(req); - - async { - let res = fut.await?; - if let Some(histogram_timer) = histogram_timer { - histogram_timer.observe_duration(); - }; - Ok(res) - } - }) + .configure(|s| configure_metrics_route(s, $opt.enable_metrics_route)) .wrap( Cors::default() .send_wildcard() @@ -203,5 +181,9 @@ macro_rules! create_app { .wrap(middleware::NormalizePath::new( middleware::TrailingSlash::Trim, )) + .wrap(Condition::new( + $opt.enable_metrics_route, + route_metrics::RouteMetrics, + )) }}; } diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 04b61f74e..0f1e356e7 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -146,6 +146,10 @@ pub struct Opt { #[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")] pub log_level: String, + // Enables Prometheus metrics and /metrics route. + #[clap(long, requires = "enable-metrics-route")] + pub enable_metrics_route: bool, + #[serde(flatten)] #[clap(flatten)] pub indexer_options: IndexerOpts, @@ -161,7 +165,7 @@ impl Opt { pub fn analytics(&self) -> bool { !self.no_analytics } - + pub fn get_ssl_config(&self) -> anyhow::Result> { if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) { let config = rustls::ServerConfig::builder().with_safe_defaults(); diff --git a/meilisearch-http/src/route_metrics.rs b/meilisearch-http/src/route_metrics.rs new file mode 100644 index 000000000..b1b85f9c8 --- /dev/null +++ b/meilisearch-http/src/route_metrics.rs @@ -0,0 +1,75 @@ +use std::future::{ready, Ready}; + +use actix_web::{ + dev::{self, Service, ServiceRequest, ServiceResponse, Transform}, + Error, +}; +use futures_util::future::LocalBoxFuture; +use prometheus::HistogramTimer; + +pub struct RouteMetrics; + +// Middleware factory is `Transform` trait from actix-service crate +// `S` - type of the next service +// `B` - type of response's body +impl Transform for RouteMetrics +where + S: Service, Error = Error>, + S::Future: 'static, + B: 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type InitError = (); + type Transform = RouteMetricsMiddleware; + type Future = Ready>; + + fn new_transform(&self, service: S) -> Self::Future { + ready(Ok(RouteMetricsMiddleware { service })) + } +} + +pub struct RouteMetricsMiddleware { + service: S, +} + +impl Service for RouteMetricsMiddleware +where + S: Service, Error = Error>, + S::Future: 'static, + B: 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type Future = LocalBoxFuture<'static, Result>; + + dev::forward_ready!(service); + + fn call(&self, req: ServiceRequest) -> Self::Future { + let mut histogram_timer: Option = None; + let request_path = req.path(); + let is_registered_resource = req.resource_map().has_resource(request_path); + if is_registered_resource { + let request_method = req.method().to_string(); + histogram_timer = Some( + crate::metrics::HTTP_RESPONSE_TIME_SECONDS + .with_label_values(&[&request_method, request_path]) + .start_timer(), + ); + crate::metrics::HTTP_REQUESTS_TOTAL + .with_label_values(&[&request_method, request_path]) + .inc(); + } + + let fut = self.service.call(req); + + Box::pin(async move { + let res = fut.await?; + + if let Some(histogram_timer) = histogram_timer { + histogram_timer.observe_duration(); + }; + Ok(res) + }) + } +} diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 8f5c9f3d4..9412dece2 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -21,7 +21,6 @@ mod tasks; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::scope("/tasks").configure(tasks::configure)) .service(web::resource("/health").route(web::get().to(get_health))) - .service(web::resource("/metrics").route(web::get().to(get_metrics))) .service(web::scope("/keys").configure(api_key::configure)) .service(web::scope("/dumps").configure(dump::configure)) .service(web::resource("/stats").route(web::get().to(get_stats))) From 947fb5c956beb7546e2fc220fe9316c273690b9c Mon Sep 17 00:00:00 2001 From: Pavo-Tusker <43550760+pavo-tusker@users.noreply.github.com> Date: Wed, 17 Aug 2022 18:49:52 +0530 Subject: [PATCH 06/20] Update meilisearch-http/src/option.rs Co-authored-by: Tamo --- meilisearch-http/src/option.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 0f1e356e7..5bc576823 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -146,8 +146,8 @@ pub struct Opt { #[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")] pub log_level: String, - // Enables Prometheus metrics and /metrics route. - #[clap(long, requires = "enable-metrics-route")] + /// Enables Prometheus metrics and /metrics route. + #[clap(long)] pub enable_metrics_route: bool, #[serde(flatten)] From 54a0b47c2b4697db269f0b924c19d69ee6cd3c30 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Wed, 17 Aug 2022 21:08:28 +0530 Subject: [PATCH 07/20] clippy issue in metrics fixed --- meilisearch-http/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index edffb6afc..632781191 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -5,6 +5,7 @@ pub mod analytics; pub mod task; #[macro_use] pub mod extractors; +pub mod metrics; pub mod option; pub mod route_metrics; pub mod routes; From 43bb5176a9ef339ccb1dc028e8e27f1615b4732e Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Mon, 22 Aug 2022 10:30:07 +0530 Subject: [PATCH 08/20] conflict fixes --- meilisearch-http/src/option.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 5bc576823..e97bf39d7 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -165,7 +165,7 @@ impl Opt { pub fn analytics(&self) -> bool { !self.no_analytics } - + pub fn get_ssl_config(&self) -> anyhow::Result> { if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) { let config = rustls::ServerConfig::builder().with_safe_defaults(); From d5da063666c32642452849a7d484933eb83b3989 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Mon, 22 Aug 2022 10:52:09 +0530 Subject: [PATCH 09/20] clippy & fmt fixed --- meilisearch-http/tests/auth/payload.rs | 10 +++++----- meilisearch-http/tests/common/service.rs | 12 ++++++------ meilisearch-http/tests/content_type.rs | 4 ++-- .../tests/documents/add_documents.rs | 18 +++++++++--------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/meilisearch-http/tests/auth/payload.rs b/meilisearch-http/tests/auth/payload.rs index ae9ac65fa..4437cd5f7 100644 --- a/meilisearch-http/tests/auth/payload.rs +++ b/meilisearch-http/tests/auth/payload.rs @@ -19,7 +19,7 @@ async fn error_api_key_bad_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -91,7 +91,7 @@ async fn error_api_key_empty_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -163,7 +163,7 @@ async fn error_api_key_missing_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -227,7 +227,7 @@ async fn error_api_key_empty_payload() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -283,7 +283,7 @@ async fn error_api_key_malformed_payload() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index e3949ce67..0834abf8d 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -18,7 +18,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -46,7 +46,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -72,7 +72,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -95,7 +95,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -118,7 +118,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -141,7 +141,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; diff --git a/meilisearch-http/tests/content_type.rs b/meilisearch-http/tests/content_type.rs index eace67a08..47e224bd1 100644 --- a/meilisearch-http/tests/content_type.rs +++ b/meilisearch-http/tests/content_type.rs @@ -63,7 +63,7 @@ async fn error_json_bad_content_type() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -146,7 +146,7 @@ async fn extract_actual_content_type() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index ebd5f58d3..b5819ad98 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -21,7 +21,7 @@ async fn add_documents_test_json_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -67,7 +67,7 @@ async fn error_add_documents_test_bad_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -135,7 +135,7 @@ async fn error_add_documents_test_no_content_type() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -195,7 +195,7 @@ async fn error_add_malformed_csv_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -257,7 +257,7 @@ async fn error_add_malformed_json_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -369,7 +369,7 @@ async fn error_add_malformed_ndjson_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -429,7 +429,7 @@ async fn error_add_missing_payload_csv_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -481,7 +481,7 @@ async fn error_add_missing_payload_json_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -533,7 +533,7 @@ async fn error_add_missing_payload_ndjson_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; From 4bee0565e8fbf8138145a03afbe48d2009ee9e05 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Thu, 4 Aug 2022 20:16:09 +0530 Subject: [PATCH 10/20] prometheus and grafana dashboards implemented --- Cargo.lock | 38 ++ grafana-dashboards/dashboard.json | 1007 ++++++++++++++++++++++++++++ meilisearch-http/Cargo.toml | 2 + meilisearch-http/src/lib.rs | 33 + meilisearch-http/src/metrics.rs | 40 ++ meilisearch-http/src/routes/mod.rs | 38 +- 6 files changed, 1157 insertions(+), 1 deletion(-) create mode 100644 grafana-dashboards/dashboard.json create mode 100644 meilisearch-http/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 2eddd276e..d565d4708 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2048,6 +2048,7 @@ dependencies = [ "indexmap", "itertools", "jsonwebtoken", + "lazy_static", "log", "manifest-dir-macros", "maplit", @@ -2061,6 +2062,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "platform-dirs", + "prometheus", "rand", "rayon", "regex", @@ -2667,6 +2669,36 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0941606b9934e2d98a3677759a971756eb821f75764d0e0d26946d08e74d9104" +dependencies = [ + "bitflags", + "byteorder", + "hex", + "lazy_static", + "libc", +] + +[[package]] +name = "prometheus" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cface98dfa6d645ea4c789839f176e4b072265d085bfcc48eaa8d137f58d3c39" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs", + "protobuf", + "thiserror", +] + [[package]] name = "proptest" version = "1.0.0" @@ -2698,6 +2730,12 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "protobuf" +version = "2.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" + [[package]] name = "quick-error" version = "1.2.3" diff --git a/grafana-dashboards/dashboard.json b/grafana-dashboards/dashboard.json new file mode 100644 index 000000000..fe64e2966 --- /dev/null +++ b/grafana-dashboards/dashboard.json @@ -0,0 +1,1007 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "i51CxikVz" + }, + "refId": "A" + } + ], + "title": "Web application metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 2, + "interval": "5s", + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_database_size{job=\"meilisearch\", instance=\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Database Size", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 22, + "interval": "5s", + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "meilisearch_total_index{job=\"meilisearch\", instance=\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Indexes Count", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 18, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "expr": "meilisearch_docs_count{job=\"meilisearch\", index=\"$Index\", instance=\"$instance\"}", + "hide": false, + "range": true, + "refId": "A" + } + ], + "title": "Total Documents", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 19, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "round(increase(http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"meilisearch\"}[1h]))", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total Searches (1h)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 20, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "round(increase(http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"meilisearch\"}[24h]))", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total Searches (24h)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 21, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.0.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "round(increase(http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"meilisearch\"}[30d]))", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total Searches (30d)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 1, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "rate(http_requests_total{instance=\"$instance\", job=\"meilisearch\"}[5m])", + "interval": "", + "legendFormat": "{{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP requests per second (All Indexes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "rate(http_response_time_seconds_sum{instance=\"$instance\", job=\"meilisearch\"}[5m]) / rate(http_response_time_seconds_count[5m])", + "interval": "", + "legendFormat": "{{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Mean response time (All Indexes)", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateBlues", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 18 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 16, + "legend": { + "show": false + }, + "pluginVersion": "8.1.4", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum by(le) (increase(http_response_time_seconds_bucket{path=\"/indexes/$Index/search\", instance=\"$instance\", job=\"meilisearch\"}[30s]))", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Response time distribution over time (`POST /indexes/:index/search`)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": 10, + "yAxis": { + "decimals": 2, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto", + "yBucketNumber": 10 + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "i51CxikVz" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "i51CxikVz" + }, + "refId": "A" + } + ], + "title": "System metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 4, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "rate(process_cpu_seconds_total{job=\"meilisearch\", instance=\"$instance\"}[1m])", + "interval": "", + "legendFormat": "process", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{name='mongodb-redis'}[1m])) by (name)", + "interval": "", + "legendFormat": "container", + "refId": "B" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisLabel": "MiB", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 5, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "process_resident_memory_bytes{job=\"meilisearch\", instance=\"$instance\"} / 1024 / 1024", + "interval": "", + "legendFormat": "process", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "exemplar": true, + "expr": "container_memory_usage_bytes{name=\"mongodb-redis\"} / 1024 / 1024", + "interval": "", + "legendFormat": "container", + "refId": "B" + } + ], + "title": "Memory usage", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "localhost:7700", + "value": "localhost:7700" + }, + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "definition": "label_values(instance)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "movie-collection", + "value": "movie-collection" + }, + "datasource": { + "type": "prometheus", + "uid": "1MRsknzVz" + }, + "definition": "label_values(index)", + "hide": 0, + "includeAll": false, + "label": "index", + "multi": false, + "name": "Index", + "options": [], + "query": { + "query": "label_values(index)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m" + ] + }, + "timezone": "", + "title": "MeiliSearch", + "uid": "7wcZ94dnz", + "version": 47, + "weekStart": "" + } \ No newline at end of file diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 81e0205da..9395a604c 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -77,6 +77,8 @@ tokio = { version = "1.17.0", features = ["full"] } tokio-stream = "0.1.8" uuid = { version = "1.1.2", features = ["serde", "v4"] } walkdir = "2.3.2" +prometheus = { version = "0.13.0", features = ["process"] } +lazy_static = "1.4.0" [dev-dependencies] actix-rt = "2.7.0" diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index fcf07587f..a4395f606 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -7,6 +7,7 @@ pub mod task; pub mod extractors; pub mod option; pub mod routes; +pub mod metrics; use std::sync::{atomic::AtomicBool, Arc}; use std::time::Duration; @@ -146,16 +147,48 @@ macro_rules! create_app { use actix_cors::Cors; use actix_web::middleware::TrailingSlash; use actix_web::App; + use actix_web::dev::Service; use actix_web::{middleware, web}; use meilisearch_http::error::MeilisearchHttpError; use meilisearch_http::routes; use meilisearch_http::{configure_data, dashboard}; use meilisearch_types::error::ResponseError; + use meilisearch_http::metrics; + + use lazy_static::lazy_static; + use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge}; + use prometheus::{HistogramVec, IntCounterVec, IntGauge, HistogramTimer}; App::new() .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics)) .configure(routes::configure) .configure(|s| dashboard(s, $enable_frontend)) + .wrap_fn(|req, srv| { + let mut histogram_timer: Option = None; + let request_path = req.path(); + let is_registered_resource = req.resource_map().has_resource(request_path); + if is_registered_resource { + let request_method = req.method().to_string(); + histogram_timer = Some( + metrics::HTTP_RESPONSE_TIME_SECONDS + .with_label_values(&[&request_method, request_path]) + .start_timer(), + ); + metrics::HTTP_REQUESTS_TOTAL + .with_label_values(&[&request_method, request_path]) + .inc(); + } + + let fut = srv.call(req); + + async { + let res = fut.await?; + if let Some(histogram_timer) = histogram_timer { + histogram_timer.observe_duration(); + }; + Ok(res) + } + }) .wrap( Cors::default() .send_wildcard() diff --git a/meilisearch-http/src/metrics.rs b/meilisearch-http/src/metrics.rs new file mode 100644 index 000000000..b93696dd3 --- /dev/null +++ b/meilisearch-http/src/metrics.rs @@ -0,0 +1,40 @@ +use lazy_static::lazy_static; +use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, register_int_gauge_vec}; +use prometheus::{HistogramVec, IntCounterVec, IntGauge, IntGaugeVec}; + +const HTTP_RESPONSE_TIME_CUSTOM_BUCKETS: &[f64; 14] = &[ + 0.0005, 0.0008, 0.00085, 0.0009, 0.00095, 0.001, 0.00105, 0.0011, 0.00115, 0.0012, 0.0015, + 0.002, 0.003, 1.0, +]; + +lazy_static! { + pub static ref HTTP_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!( + opts!("http_requests_total", "HTTP requests total"), + &["method", "path"] + ) + .expect("Can't create a metric"); + + pub static ref MEILISEARCH_DB_SIZE: IntGauge = register_int_gauge!( + opts!("meilisearch_database_size", "MeiliSearch Stats DbSize") + ) + .expect("Can't create a metric"); + + pub static ref MEILISEARCH_INDEX_COUNT: IntGauge = register_int_gauge!( + opts!("meilisearch_total_index", "MeiliSearch Stats Index Count") + ) + .expect("Can't create a metric"); + + pub static ref MEILISEARCH_DOCS_COUNT: IntGaugeVec = register_int_gauge_vec!( + opts!("meilisearch_docs_count", "MeiliSearch Stats Docs Count"), + &["index"] + ) + .expect("Can't create a metric"); + + pub static ref HTTP_RESPONSE_TIME_SECONDS: HistogramVec = register_histogram_vec!( + "http_response_time_seconds", + "HTTP response times", + &["method", "path"], + HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec() + ) + .expect("Can't create a metric"); +} \ No newline at end of file diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 6a673f600..e588189ce 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,4 +1,5 @@ -use actix_web::{web, HttpRequest, HttpResponse}; +use actix_web::{web, HttpResponse}; +use actix_web::http::header::{self}; use log::debug; use serde::{Deserialize, Serialize}; @@ -12,6 +13,7 @@ use meilisearch_types::star_or::StarOr; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; +use prometheus::{Encoder, TextEncoder}; mod api_key; mod dump; @@ -21,6 +23,7 @@ mod tasks; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::scope("/tasks").configure(tasks::configure)) .service(web::resource("/health").route(web::get().to(get_health))) + .service(web::resource("/metrics").route(web::get().to(get_metrics))) .service(web::scope("/keys").configure(api_key::configure)) .service(web::scope("/dumps").configure(dump::configure)) .service(web::resource("/stats").route(web::get().to(get_stats))) @@ -278,3 +281,36 @@ struct KeysResponse { pub async fn get_health() -> Result { Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) } + +pub async fn get_metrics( + meilisearch: GuardedData, MeiliSearch>, +) -> Result { + + let search_rules = &meilisearch.filters().search_rules; + let response = meilisearch.get_all_stats(search_rules).await?; + + crate::metrics::MEILISEARCH_DB_SIZE + .set(response.database_size as i64); + + crate::metrics::MEILISEARCH_INDEX_COUNT + .set(response.indexes.len() as i64); + + for (index, value) in response.indexes.iter() { + crate::metrics::MEILISEARCH_DOCS_COUNT + .with_label_values(&[&index]) + .set(value.number_of_documents as i64); + } + + let encoder = TextEncoder::new(); + let mut buffer = vec![]; + encoder + .encode(&prometheus::gather(), &mut buffer) + .expect("Failed to encode metrics"); + + let response = String::from_utf8(buffer.clone()).expect("Failed to convert bytes to string"); + buffer.clear(); + + Ok(HttpResponse::Ok() + .insert_header(header::ContentType(mime::TEXT_PLAIN)) + .body(response)) +} From 25fecf93604eb7ffb7dae50539cde3bfbb7b62a2 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Mon, 8 Aug 2022 11:11:38 +0530 Subject: [PATCH 11/20] clippy & rustfmt fixed --- meilisearch-http/src/lib.rs | 11 ++++++----- meilisearch-http/src/metrics.rs | 25 +++++++++++++------------ meilisearch-http/src/routes/mod.rs | 13 +++++-------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index a4395f606..f0ce0396c 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -7,7 +7,6 @@ pub mod task; pub mod extractors; pub mod option; pub mod routes; -pub mod metrics; use std::sync::{atomic::AtomicBool, Arc}; use std::time::Duration; @@ -145,19 +144,21 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { macro_rules! create_app { ($data:expr, $auth:expr, $enable_frontend:expr, $opt:expr, $analytics:expr) => {{ use actix_cors::Cors; + use actix_web::dev::Service; use actix_web::middleware::TrailingSlash; use actix_web::App; - use actix_web::dev::Service; use actix_web::{middleware, web}; use meilisearch_http::error::MeilisearchHttpError; + use meilisearch_http::metrics; use meilisearch_http::routes; use meilisearch_http::{configure_data, dashboard}; use meilisearch_types::error::ResponseError; - use meilisearch_http::metrics; use lazy_static::lazy_static; - use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge}; - use prometheus::{HistogramVec, IntCounterVec, IntGauge, HistogramTimer}; + use prometheus::{ + opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, + }; + use prometheus::{HistogramTimer, HistogramVec, IntCounterVec, IntGauge}; App::new() .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics)) diff --git a/meilisearch-http/src/metrics.rs b/meilisearch-http/src/metrics.rs index b93696dd3..8786d7aeb 100644 --- a/meilisearch-http/src/metrics.rs +++ b/meilisearch-http/src/metrics.rs @@ -1,5 +1,8 @@ use lazy_static::lazy_static; -use prometheus::{opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, register_int_gauge_vec}; +use prometheus::{ + opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, + register_int_gauge_vec, +}; use prometheus::{HistogramVec, IntCounterVec, IntGauge, IntGaugeVec}; const HTTP_RESPONSE_TIME_CUSTOM_BUCKETS: &[f64; 14] = &[ @@ -13,23 +16,21 @@ lazy_static! { &["method", "path"] ) .expect("Can't create a metric"); - - pub static ref MEILISEARCH_DB_SIZE: IntGauge = register_int_gauge!( - opts!("meilisearch_database_size", "MeiliSearch Stats DbSize") - ) + pub static ref MEILISEARCH_DB_SIZE: IntGauge = register_int_gauge!(opts!( + "meilisearch_database_size", + "MeiliSearch Stats DbSize" + )) .expect("Can't create a metric"); - - pub static ref MEILISEARCH_INDEX_COUNT: IntGauge = register_int_gauge!( - opts!("meilisearch_total_index", "MeiliSearch Stats Index Count") - ) + pub static ref MEILISEARCH_INDEX_COUNT: IntGauge = register_int_gauge!(opts!( + "meilisearch_total_index", + "MeiliSearch Stats Index Count" + )) .expect("Can't create a metric"); - pub static ref MEILISEARCH_DOCS_COUNT: IntGaugeVec = register_int_gauge_vec!( opts!("meilisearch_docs_count", "MeiliSearch Stats Docs Count"), &["index"] ) .expect("Can't create a metric"); - pub static ref HTTP_RESPONSE_TIME_SECONDS: HistogramVec = register_histogram_vec!( "http_response_time_seconds", "HTTP response times", @@ -37,4 +38,4 @@ lazy_static! { HTTP_RESPONSE_TIME_CUSTOM_BUCKETS.to_vec() ) .expect("Can't create a metric"); -} \ No newline at end of file +} diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index e588189ce..8c387f2af 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,5 +1,5 @@ -use actix_web::{web, HttpResponse}; use actix_web::http::header::{self}; +use actix_web::{web, HttpResponse}; use log::debug; use serde::{Deserialize, Serialize}; @@ -285,20 +285,17 @@ pub async fn get_health() -> Result { pub async fn get_metrics( meilisearch: GuardedData, MeiliSearch>, ) -> Result { - let search_rules = &meilisearch.filters().search_rules; let response = meilisearch.get_all_stats(search_rules).await?; - crate::metrics::MEILISEARCH_DB_SIZE - .set(response.database_size as i64); + crate::metrics::MEILISEARCH_DB_SIZE.set(response.database_size as i64); - crate::metrics::MEILISEARCH_INDEX_COUNT - .set(response.indexes.len() as i64); + crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64); for (index, value) in response.indexes.iter() { crate::metrics::MEILISEARCH_DOCS_COUNT - .with_label_values(&[&index]) - .set(value.number_of_documents as i64); + .with_label_values(&[index]) + .set(value.number_of_documents as i64); } let encoder = TextEncoder::new(); From 1f1482e97cfe1c13cfb4d150dffd80a339705689 Mon Sep 17 00:00:00 2001 From: Pavo-Tusker <43550760+pavo-tusker@users.noreply.github.com> Date: Thu, 11 Aug 2022 11:38:06 +0530 Subject: [PATCH 12/20] Update meilisearch-http/src/routes/mod.rs Co-authored-by: Tamo --- meilisearch-http/src/routes/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 8c387f2af..e27141875 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -289,7 +289,6 @@ pub async fn get_metrics( let response = meilisearch.get_all_stats(search_rules).await?; crate::metrics::MEILISEARCH_DB_SIZE.set(response.database_size as i64); - crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64); for (index, value) in response.indexes.iter() { From 0b6ca7379008ae1ddf10af1c25d8806a8048543f Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Fri, 12 Aug 2022 12:44:14 +0530 Subject: [PATCH 13/20] review fixes --- meilisearch-http/src/routes/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index e27141875..594cbc48e 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -303,8 +303,7 @@ pub async fn get_metrics( .encode(&prometheus::gather(), &mut buffer) .expect("Failed to encode metrics"); - let response = String::from_utf8(buffer.clone()).expect("Failed to convert bytes to string"); - buffer.clear(); + let response = String::from_utf8(buffer).expect("Failed to convert bytes to string"); Ok(HttpResponse::Ok() .insert_header(header::ContentType(mime::TEXT_PLAIN)) From 2b8f3c26ece55cb1f701b38adbfba13f35d131c3 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Wed, 17 Aug 2022 17:14:55 +0530 Subject: [PATCH 14/20] Changed prometheus metrics feature as optional --- meilisearch-http/src/lib.rs | 48 ++++++----------- meilisearch-http/src/option.rs | 6 ++- meilisearch-http/src/route_metrics.rs | 75 +++++++++++++++++++++++++++ meilisearch-http/src/routes/mod.rs | 1 - 4 files changed, 95 insertions(+), 35 deletions(-) create mode 100644 meilisearch-http/src/route_metrics.rs diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index f0ce0396c..edffb6afc 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -6,6 +6,7 @@ pub mod task; #[macro_use] pub mod extractors; pub mod option; +pub mod route_metrics; pub mod routes; use std::sync::{atomic::AtomicBool, Arc}; @@ -140,56 +141,33 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { config.service(web::resource("/").route(web::get().to(routes::running))); } +pub fn configure_metrics_route(config: &mut web::ServiceConfig, enable_metrics_route: bool) { + if enable_metrics_route { + config.service(web::resource("/metrics").route(web::get().to(routes::get_metrics))); + } +} + #[macro_export] macro_rules! create_app { ($data:expr, $auth:expr, $enable_frontend:expr, $opt:expr, $analytics:expr) => {{ use actix_cors::Cors; use actix_web::dev::Service; + use actix_web::middleware::Condition; use actix_web::middleware::TrailingSlash; use actix_web::App; use actix_web::{middleware, web}; use meilisearch_http::error::MeilisearchHttpError; use meilisearch_http::metrics; + use meilisearch_http::route_metrics; use meilisearch_http::routes; - use meilisearch_http::{configure_data, dashboard}; + use meilisearch_http::{configure_data, configure_metrics_route, dashboard}; use meilisearch_types::error::ResponseError; - use lazy_static::lazy_static; - use prometheus::{ - opts, register_histogram_vec, register_int_counter_vec, register_int_gauge, - }; - use prometheus::{HistogramTimer, HistogramVec, IntCounterVec, IntGauge}; - App::new() .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics)) .configure(routes::configure) .configure(|s| dashboard(s, $enable_frontend)) - .wrap_fn(|req, srv| { - let mut histogram_timer: Option = None; - let request_path = req.path(); - let is_registered_resource = req.resource_map().has_resource(request_path); - if is_registered_resource { - let request_method = req.method().to_string(); - histogram_timer = Some( - metrics::HTTP_RESPONSE_TIME_SECONDS - .with_label_values(&[&request_method, request_path]) - .start_timer(), - ); - metrics::HTTP_REQUESTS_TOTAL - .with_label_values(&[&request_method, request_path]) - .inc(); - } - - let fut = srv.call(req); - - async { - let res = fut.await?; - if let Some(histogram_timer) = histogram_timer { - histogram_timer.observe_duration(); - }; - Ok(res) - } - }) + .configure(|s| configure_metrics_route(s, $opt.enable_metrics_route)) .wrap( Cors::default() .send_wildcard() @@ -203,5 +181,9 @@ macro_rules! create_app { .wrap(middleware::NormalizePath::new( middleware::TrailingSlash::Trim, )) + .wrap(Condition::new( + $opt.enable_metrics_route, + route_metrics::RouteMetrics, + )) }}; } diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 04b61f74e..0f1e356e7 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -146,6 +146,10 @@ pub struct Opt { #[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")] pub log_level: String, + // Enables Prometheus metrics and /metrics route. + #[clap(long, requires = "enable-metrics-route")] + pub enable_metrics_route: bool, + #[serde(flatten)] #[clap(flatten)] pub indexer_options: IndexerOpts, @@ -161,7 +165,7 @@ impl Opt { pub fn analytics(&self) -> bool { !self.no_analytics } - + pub fn get_ssl_config(&self) -> anyhow::Result> { if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) { let config = rustls::ServerConfig::builder().with_safe_defaults(); diff --git a/meilisearch-http/src/route_metrics.rs b/meilisearch-http/src/route_metrics.rs new file mode 100644 index 000000000..b1b85f9c8 --- /dev/null +++ b/meilisearch-http/src/route_metrics.rs @@ -0,0 +1,75 @@ +use std::future::{ready, Ready}; + +use actix_web::{ + dev::{self, Service, ServiceRequest, ServiceResponse, Transform}, + Error, +}; +use futures_util::future::LocalBoxFuture; +use prometheus::HistogramTimer; + +pub struct RouteMetrics; + +// Middleware factory is `Transform` trait from actix-service crate +// `S` - type of the next service +// `B` - type of response's body +impl Transform for RouteMetrics +where + S: Service, Error = Error>, + S::Future: 'static, + B: 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type InitError = (); + type Transform = RouteMetricsMiddleware; + type Future = Ready>; + + fn new_transform(&self, service: S) -> Self::Future { + ready(Ok(RouteMetricsMiddleware { service })) + } +} + +pub struct RouteMetricsMiddleware { + service: S, +} + +impl Service for RouteMetricsMiddleware +where + S: Service, Error = Error>, + S::Future: 'static, + B: 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type Future = LocalBoxFuture<'static, Result>; + + dev::forward_ready!(service); + + fn call(&self, req: ServiceRequest) -> Self::Future { + let mut histogram_timer: Option = None; + let request_path = req.path(); + let is_registered_resource = req.resource_map().has_resource(request_path); + if is_registered_resource { + let request_method = req.method().to_string(); + histogram_timer = Some( + crate::metrics::HTTP_RESPONSE_TIME_SECONDS + .with_label_values(&[&request_method, request_path]) + .start_timer(), + ); + crate::metrics::HTTP_REQUESTS_TOTAL + .with_label_values(&[&request_method, request_path]) + .inc(); + } + + let fut = self.service.call(req); + + Box::pin(async move { + let res = fut.await?; + + if let Some(histogram_timer) = histogram_timer { + histogram_timer.observe_duration(); + }; + Ok(res) + }) + } +} diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 594cbc48e..e6c11b8d7 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -23,7 +23,6 @@ mod tasks; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::scope("/tasks").configure(tasks::configure)) .service(web::resource("/health").route(web::get().to(get_health))) - .service(web::resource("/metrics").route(web::get().to(get_metrics))) .service(web::scope("/keys").configure(api_key::configure)) .service(web::scope("/dumps").configure(dump::configure)) .service(web::resource("/stats").route(web::get().to(get_stats))) From a58b00d8f1d76c4d870917a968099eeec1937acf Mon Sep 17 00:00:00 2001 From: Pavo-Tusker <43550760+pavo-tusker@users.noreply.github.com> Date: Wed, 17 Aug 2022 18:49:52 +0530 Subject: [PATCH 15/20] Update meilisearch-http/src/option.rs Co-authored-by: Tamo --- meilisearch-http/src/option.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 0f1e356e7..5bc576823 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -146,8 +146,8 @@ pub struct Opt { #[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")] pub log_level: String, - // Enables Prometheus metrics and /metrics route. - #[clap(long, requires = "enable-metrics-route")] + /// Enables Prometheus metrics and /metrics route. + #[clap(long)] pub enable_metrics_route: bool, #[serde(flatten)] From 2f92169e48a878222bc729b589c49c93e9b273ef Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Wed, 17 Aug 2022 21:08:28 +0530 Subject: [PATCH 16/20] clippy issue in metrics fixed --- meilisearch-http/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index edffb6afc..632781191 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -5,6 +5,7 @@ pub mod analytics; pub mod task; #[macro_use] pub mod extractors; +pub mod metrics; pub mod option; pub mod route_metrics; pub mod routes; From 109540011a5101b1906b98af37de230bfffd6955 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Mon, 22 Aug 2022 10:30:07 +0530 Subject: [PATCH 17/20] conflict fixes --- meilisearch-http/src/option.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 5bc576823..e97bf39d7 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -165,7 +165,7 @@ impl Opt { pub fn analytics(&self) -> bool { !self.no_analytics } - + pub fn get_ssl_config(&self) -> anyhow::Result> { if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) { let config = rustls::ServerConfig::builder().with_safe_defaults(); From a05101af4db53510ed39c2b65561466c59cb19a7 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Mon, 22 Aug 2022 10:52:09 +0530 Subject: [PATCH 18/20] clippy & fmt fixed --- meilisearch-http/tests/auth/payload.rs | 10 +++++----- meilisearch-http/tests/common/service.rs | 12 ++++++------ meilisearch-http/tests/content_type.rs | 4 ++-- .../tests/documents/add_documents.rs | 18 +++++++++--------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/meilisearch-http/tests/auth/payload.rs b/meilisearch-http/tests/auth/payload.rs index ae9ac65fa..4437cd5f7 100644 --- a/meilisearch-http/tests/auth/payload.rs +++ b/meilisearch-http/tests/auth/payload.rs @@ -19,7 +19,7 @@ async fn error_api_key_bad_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -91,7 +91,7 @@ async fn error_api_key_empty_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -163,7 +163,7 @@ async fn error_api_key_missing_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -227,7 +227,7 @@ async fn error_api_key_empty_payload() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -283,7 +283,7 @@ async fn error_api_key_malformed_payload() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; diff --git a/meilisearch-http/tests/common/service.rs b/meilisearch-http/tests/common/service.rs index e3949ce67..0834abf8d 100644 --- a/meilisearch-http/tests/common/service.rs +++ b/meilisearch-http/tests/common/service.rs @@ -18,7 +18,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -46,7 +46,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -72,7 +72,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -95,7 +95,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -118,7 +118,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; @@ -141,7 +141,7 @@ impl Service { &self.meilisearch, &self.auth, true, - &self.options, + self.options, analytics::MockAnalytics::new(&self.options).0 )) .await; diff --git a/meilisearch-http/tests/content_type.rs b/meilisearch-http/tests/content_type.rs index eace67a08..47e224bd1 100644 --- a/meilisearch-http/tests/content_type.rs +++ b/meilisearch-http/tests/content_type.rs @@ -63,7 +63,7 @@ async fn error_json_bad_content_type() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -146,7 +146,7 @@ async fn extract_actual_content_type() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index afee5b18c..909ea873a 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -21,7 +21,7 @@ async fn add_documents_test_json_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -66,7 +66,7 @@ async fn add_single_document_test_json_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -112,7 +112,7 @@ async fn error_add_documents_test_bad_content_types() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -180,7 +180,7 @@ async fn error_add_documents_test_no_content_type() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -240,7 +240,7 @@ async fn error_add_malformed_csv_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -414,7 +414,7 @@ async fn error_add_malformed_ndjson_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -474,7 +474,7 @@ async fn error_add_missing_payload_csv_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -526,7 +526,7 @@ async fn error_add_missing_payload_json_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; @@ -578,7 +578,7 @@ async fn error_add_missing_payload_ndjson_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await; From ef1d6b1694d027a23ed36ea1a039d2586c0046b5 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Mon, 22 Aug 2022 13:27:26 +0530 Subject: [PATCH 19/20] clippy & cargo fixed --- meilisearch-http/src/routes/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index e6c11b8d7..598dae42b 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,5 +1,5 @@ use actix_web::http::header::{self}; -use actix_web::{web, HttpResponse}; +use actix_web::{web, HttpRequest, HttpResponse}; use log::debug; use serde::{Deserialize, Serialize}; From a37c7ba1bb57c69b8b1ce00820c1c96bab3ef5f4 Mon Sep 17 00:00:00 2001 From: mohandasspat Date: Mon, 22 Aug 2022 13:34:19 +0530 Subject: [PATCH 20/20] clippy & cargo fixed --- meilisearch-http/tests/documents/add_documents.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index 909ea873a..685428784 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -302,7 +302,7 @@ async fn error_add_malformed_json_documents() { &server.service.meilisearch, &server.service.auth, true, - &server.service.options, + server.service.options, analytics::MockAnalytics::new(&server.service.options).0 )) .await;