move prometheus behind a feature flag

This commit is contained in:
Tamo 2022-08-29 12:36:54 +02:00 committed by Kerollmops
parent 47a1aa69f0
commit 6aa3ad6b6c
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
7 changed files with 79 additions and 47 deletions

View File

@ -78,7 +78,7 @@ tokio = { version = "1.17.0", features = ["full"] }
tokio-stream = "0.1.8" tokio-stream = "0.1.8"
uuid = { version = "1.1.2", features = ["serde", "v4"] } uuid = { version = "1.1.2", features = ["serde", "v4"] }
walkdir = "2.3.2" walkdir = "2.3.2"
prometheus = { version = "0.13.0", features = ["process"] } prometheus = { version = "0.13.0", features = ["process"], optional = true }
lazy_static = "1.4.0" lazy_static = "1.4.0"
[dev-dependencies] [dev-dependencies]
@ -91,6 +91,7 @@ yaup = "0.2.0"
[features] [features]
default = ["analytics", "mini-dashboard"] default = ["analytics", "mini-dashboard"]
metrics = ["prometheus"]
analytics = ["segment"] analytics = ["segment"]
mini-dashboard = [ mini-dashboard = [
"actix-web-static-files", "actix-web-static-files",

View File

@ -5,11 +5,14 @@ pub mod analytics;
pub mod task; pub mod task;
#[macro_use] #[macro_use]
pub mod extractors; pub mod extractors;
pub mod metrics;
pub mod option; pub mod option;
pub mod route_metrics;
pub mod routes; pub mod routes;
#[cfg(feature = "metrics")]
pub mod metrics;
#[cfg(feature = "metrics")]
pub mod route_metrics;
use std::sync::{atomic::AtomicBool, Arc}; use std::sync::{atomic::AtomicBool, Arc};
use std::time::Duration; use std::time::Duration;
@ -142,9 +145,12 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) {
config.service(web::resource("/").route(web::get().to(routes::running))); config.service(web::resource("/").route(web::get().to(routes::running)));
} }
#[cfg(feature = "metrics")]
pub fn configure_metrics_route(config: &mut web::ServiceConfig, enable_metrics_route: bool) { pub fn configure_metrics_route(config: &mut web::ServiceConfig, enable_metrics_route: bool) {
if enable_metrics_route { if enable_metrics_route {
config.service(web::resource("/metrics").route(web::get().to(routes::get_metrics))); config.service(
web::resource("/metrics").route(web::get().to(crate::route_metrics::get_metrics)),
);
} }
} }
@ -158,17 +164,21 @@ macro_rules! create_app {
use actix_web::App; use actix_web::App;
use actix_web::{middleware, web}; use actix_web::{middleware, web};
use meilisearch_http::error::MeilisearchHttpError; use meilisearch_http::error::MeilisearchHttpError;
use meilisearch_http::metrics;
use meilisearch_http::route_metrics;
use meilisearch_http::routes; use meilisearch_http::routes;
use meilisearch_http::{configure_data, configure_metrics_route, dashboard}; use meilisearch_http::{configure_data, dashboard};
#[cfg(feature = "metrics")]
use meilisearch_http::{configure_metrics_route, metrics, route_metrics};
use meilisearch_types::error::ResponseError; use meilisearch_types::error::ResponseError;
App::new() let app = App::new()
.configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics)) .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics))
.configure(routes::configure) .configure(routes::configure)
.configure(|s| dashboard(s, $enable_frontend)) .configure(|s| dashboard(s, $enable_frontend));
.configure(|s| configure_metrics_route(s, $opt.enable_metrics_route))
#[cfg(feature = "metrics")]
let app = app.configure(|s| configure_metrics_route(s, $opt.enable_metrics_route));
let app = app
.wrap( .wrap(
Cors::default() Cors::default()
.send_wildcard() .send_wildcard()
@ -181,10 +191,14 @@ macro_rules! create_app {
.wrap(middleware::Compress::default()) .wrap(middleware::Compress::default())
.wrap(middleware::NormalizePath::new( .wrap(middleware::NormalizePath::new(
middleware::TrailingSlash::Trim, middleware::TrailingSlash::Trim,
)) ));
.wrap(Condition::new(
#[cfg(feature = "metrics")]
let app = app.wrap(Condition::new(
$opt.enable_metrics_route, $opt.enable_metrics_route,
route_metrics::RouteMetrics, route_metrics::RouteMetrics,
)) ));
app
}}; }};
} }

View File

@ -147,6 +147,7 @@ pub struct Opt {
pub log_level: String, pub log_level: String,
/// Enables Prometheus metrics and /metrics route. /// Enables Prometheus metrics and /metrics route.
#[cfg(feature = "metrics")]
#[clap(long, env = "MEILI_ENABLE_METRICS_ROUTE")] #[clap(long, env = "MEILI_ENABLE_METRICS_ROUTE")]
pub enable_metrics_route: bool, pub enable_metrics_route: bool,

View File

@ -1,11 +1,48 @@
use std::future::{ready, Ready}; use std::future::{ready, Ready};
use actix_web::http::header;
use actix_web::HttpResponse;
use actix_web::{ use actix_web::{
dev::{self, Service, ServiceRequest, ServiceResponse, Transform}, dev::{self, Service, ServiceRequest, ServiceResponse, Transform},
Error, Error,
}; };
use futures_util::future::LocalBoxFuture; use futures_util::future::LocalBoxFuture;
use meilisearch_auth::actions;
use meilisearch_lib::MeiliSearch;
use meilisearch_types::error::ResponseError;
use prometheus::HistogramTimer; use prometheus::HistogramTimer;
use prometheus::{Encoder, TextEncoder};
use crate::extractors::authentication::policies::ActionPolicy;
use crate::extractors::authentication::GuardedData;
pub async fn get_metrics(
meilisearch: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, MeiliSearch>,
) -> Result<HttpResponse, ResponseError> {
let search_rules = &meilisearch.filters().search_rules;
let response = meilisearch.get_all_stats(search_rules).await?;
crate::metrics::MEILISEARCH_DB_SIZE_BYTES.set(response.database_size as i64);
crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64);
for (index, value) in response.indexes.iter() {
crate::metrics::MEILISEARCH_INDEX_DOCS_COUNT
.with_label_values(&[index])
.set(value.number_of_documents as i64);
}
let encoder = TextEncoder::new();
let mut buffer = vec![];
encoder
.encode(&prometheus::gather(), &mut buffer)
.expect("Failed to encode metrics");
let response = String::from_utf8(buffer).expect("Failed to convert bytes to string");
Ok(HttpResponse::Ok()
.insert_header(header::ContentType(mime::TEXT_PLAIN))
.body(response))
}
pub struct RouteMetrics; pub struct RouteMetrics;

View File

@ -1,4 +1,3 @@
use actix_web::http::header::{self};
use actix_web::{web, HttpRequest, HttpResponse}; use actix_web::{web, HttpRequest, HttpResponse};
use log::debug; use log::debug;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -13,7 +12,6 @@ use meilisearch_types::star_or::StarOr;
use crate::analytics::Analytics; use crate::analytics::Analytics;
use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::authentication::{policies::*, GuardedData};
use prometheus::{Encoder, TextEncoder};
mod api_key; mod api_key;
mod dump; mod dump;
@ -280,31 +278,3 @@ struct KeysResponse {
pub async fn get_health() -> Result<HttpResponse, ResponseError> { pub async fn get_health() -> Result<HttpResponse, ResponseError> {
Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" }))) Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" })))
} }
pub async fn get_metrics(
meilisearch: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, MeiliSearch>,
) -> Result<HttpResponse, ResponseError> {
let search_rules = &meilisearch.filters().search_rules;
let response = meilisearch.get_all_stats(search_rules).await?;
crate::metrics::MEILISEARCH_DB_SIZE_BYTES.set(response.database_size as i64);
crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64);
for (index, value) in response.indexes.iter() {
crate::metrics::MEILISEARCH_INDEX_DOCS_COUNT
.with_label_values(&[index])
.set(value.number_of_documents as i64);
}
let encoder = TextEncoder::new();
let mut buffer = vec![];
encoder
.encode(&prometheus::gather(), &mut buffer)
.expect("Failed to encode metrics");
let response = String::from_utf8(buffer).expect("Failed to convert bytes to string");
Ok(HttpResponse::Ok()
.insert_header(header::ContentType(mime::TEXT_PLAIN))
.body(response))
}

View File

@ -8,7 +8,7 @@ use time::{Duration, OffsetDateTime};
pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'static str>>> = pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'static str>>> =
Lazy::new(|| { Lazy::new(|| {
hashmap! { let mut authorizations = hashmap! {
("POST", "/indexes/products/search") => hashset!{"search", "*"}, ("POST", "/indexes/products/search") => hashset!{"search", "*"},
("GET", "/indexes/products/search") => hashset!{"search", "*"}, ("GET", "/indexes/products/search") => hashset!{"search", "*"},
("POST", "/indexes/products/documents") => hashset!{"documents.add", "documents.*", "*"}, ("POST", "/indexes/products/documents") => hashset!{"documents.add", "documents.*", "*"},
@ -45,7 +45,6 @@ pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'
("PUT", "/indexes/products/settings/synonyms") => hashset!{"settings.update", "settings.*", "*"}, ("PUT", "/indexes/products/settings/synonyms") => hashset!{"settings.update", "settings.*", "*"},
("GET", "/indexes/products/stats") => hashset!{"stats.get", "stats.*", "*"}, ("GET", "/indexes/products/stats") => hashset!{"stats.get", "stats.*", "*"},
("GET", "/stats") => hashset!{"stats.get", "stats.*", "*"}, ("GET", "/stats") => hashset!{"stats.get", "stats.*", "*"},
("GET", "/metrics") => hashset!{"metrics.get", "metrics.*", "*"},
("POST", "/dumps") => hashset!{"dumps.create", "dumps.*", "*"}, ("POST", "/dumps") => hashset!{"dumps.create", "dumps.*", "*"},
("GET", "/version") => hashset!{"version", "*"}, ("GET", "/version") => hashset!{"version", "*"},
("PATCH", "/keys/mykey/") => hashset!{"keys.update", "*"}, ("PATCH", "/keys/mykey/") => hashset!{"keys.update", "*"},
@ -53,7 +52,16 @@ pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'
("DELETE", "/keys/mykey/") => hashset!{"keys.delete", "*"}, ("DELETE", "/keys/mykey/") => hashset!{"keys.delete", "*"},
("POST", "/keys") => hashset!{"keys.create", "*"}, ("POST", "/keys") => hashset!{"keys.create", "*"},
("GET", "/keys") => hashset!{"keys.get", "*"}, ("GET", "/keys") => hashset!{"keys.get", "*"},
};
if cfg!(feature = "metrics") {
authorizations.insert(
("GET", "/metrics"),
hashset! {"metrics.get", "metrics.*", "*"},
);
} }
authorizations
}); });
pub static ALL_ACTIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| { pub static ALL_ACTIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| {

View File

@ -162,6 +162,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
max_indexing_memory: MaxMemory::unlimited(), max_indexing_memory: MaxMemory::unlimited(),
..Parser::parse_from(None as Option<&str>) ..Parser::parse_from(None as Option<&str>)
}, },
#[cfg(feature = "metrics")]
enable_metrics_route: true, enable_metrics_route: true,
..Parser::parse_from(None as Option<&str>) ..Parser::parse_from(None as Option<&str>)
} }