From 68e30214ca01767d75d2c70ebd8ea6b287a454ab Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 21 Feb 2023 18:18:47 +0100 Subject: [PATCH] remove the feature flag and reorganize the module slightly --- meilisearch/Cargo.toml | 3 +- .../src/analytics/segment_analytics.rs | 3 +- meilisearch/src/lib.rs | 35 +++++----------- .../src/{route_metrics.rs => middleware.rs} | 42 ++----------------- meilisearch/src/option.rs | 11 +---- meilisearch/src/routes/metrics.rs | 42 +++++++++++++++++++ meilisearch/src/routes/mod.rs | 7 +++- meilisearch/tests/auth/authorization.rs | 5 +-- meilisearch/tests/common/server.rs | 1 - 9 files changed, 67 insertions(+), 82 deletions(-) rename meilisearch/src/{route_metrics.rs => middleware.rs} (58%) create mode 100644 meilisearch/src/routes/metrics.rs diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index 324081880..809975ec7 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -52,7 +52,7 @@ parking_lot = "0.12.1" permissive-json-pointer = { path = "../permissive-json-pointer" } pin-project-lite = "0.2.9" platform-dirs = "0.3.0" -prometheus = { version = "0.13.2", features = ["process"], optional = true } +prometheus = { version = "0.13.2", features = ["process"] } rand = "0.8.5" rayon = "1.5.3" regex = "1.6.0" @@ -107,7 +107,6 @@ zip = { version = "0.6.2", optional = true } [features] default = ["analytics", "meilisearch-types/default", "mini-dashboard"] -metrics = ["prometheus"] analytics = ["segment"] mini-dashboard = ["actix-web-static-files", "static-files", "anyhow", "cargo_toml", "hex", "reqwest", "sha-1", "tempfile", "zip"] chinese = ["meilisearch-types/chinese"] diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 92ac4b1d5..a91ca9c96 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -255,8 +255,7 @@ impl From for Infos { // to add analytics when we add a field in the Opt. // Thus we must not insert `..` at the end. let Opt { - #[cfg(features = "metrics")] - enable_metrics_route: _, + enable_metrics_route: _, db_path, http_addr, master_key: _, diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index 71e7486f1..78841e4bd 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -4,15 +4,12 @@ pub mod error; pub mod analytics; #[macro_use] pub mod extractors; +pub mod metrics; +pub mod middleware; pub mod option; pub mod routes; pub mod search; -#[cfg(feature = "metrics")] -pub mod metrics; -#[cfg(feature = "metrics")] -pub mod route_metrics; - use std::fs::File; use std::io::{BufReader, BufWriter}; use std::path::Path; @@ -25,7 +22,7 @@ use actix_http::body::MessageBody; use actix_web::dev::{ServiceFactory, ServiceResponse}; use actix_web::error::JsonPayloadError; use actix_web::web::Data; -use actix_web::{middleware, web, HttpRequest}; +use actix_web::{web, HttpRequest}; use analytics::Analytics; use anyhow::bail; use error::PayloadError; @@ -86,14 +83,13 @@ pub fn create_app( analytics.clone(), ) }) - .configure(routes::configure) + .configure(|cfg| routes::configure(cfg, opt.enable_metrics_route)) .configure(|s| dashboard(s, enable_dashboard)); - #[cfg(feature = "metrics")] - let app = app.configure(|s| configure_metrics_route(s, opt.enable_metrics_route)); - #[cfg(feature = "metrics")] - let app = - app.wrap(middleware::Condition::new(opt.enable_metrics_route, route_metrics::RouteMetrics)); + let app = app.wrap(actix_web::middleware::Condition::new( + opt.enable_metrics_route, + middleware::RouteMetrics, + )); app.wrap( Cors::default() .send_wildcard() @@ -102,9 +98,9 @@ pub fn create_app( .allow_any_method() .max_age(86_400), // 24h ) - .wrap(middleware::Logger::default()) - .wrap(middleware::Compress::default()) - .wrap(middleware::NormalizePath::new(middleware::TrailingSlash::Trim)) + .wrap(actix_web::middleware::Logger::default()) + .wrap(actix_web::middleware::Compress::default()) + .wrap(actix_web::middleware::NormalizePath::new(actix_web::middleware::TrailingSlash::Trim)) } enum OnFailure { @@ -420,15 +416,6 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) { config.service(web::resource("/").route(web::get().to(routes::running))); } -#[cfg(feature = "metrics")] -pub fn configure_metrics_route(config: &mut web::ServiceConfig, enable_metrics_route: bool) { - if enable_metrics_route { - config.service( - web::resource("/metrics").route(web::get().to(crate::route_metrics::get_metrics)), - ); - } -} - /// Parses the output of /// [`VERGEN_GIT_SEMVER_LIGHTWEIGHT`](https://docs.rs/vergen/latest/vergen/struct.Git.html#instructions) /// as a prototype name. diff --git a/meilisearch/src/route_metrics.rs b/meilisearch/src/middleware.rs similarity index 58% rename from meilisearch/src/route_metrics.rs rename to meilisearch/src/middleware.rs index a23c2b93d..080a52634 100644 --- a/meilisearch/src/route_metrics.rs +++ b/meilisearch/src/middleware.rs @@ -1,45 +1,11 @@ +//! Contains all the custom middleware used in meilisearch + use std::future::{ready, Ready}; use actix_web::dev::{self, Service, ServiceRequest, ServiceResponse, Transform}; -use actix_web::http::header; -use actix_web::web::Data; -use actix_web::{Error, HttpResponse}; +use actix_web::Error; use futures_util::future::LocalBoxFuture; -use index_scheduler::IndexScheduler; -use meilisearch_auth::AuthController; -use meilisearch_types::error::ResponseError; -use meilisearch_types::keys::actions; -use prometheus::{Encoder, HistogramTimer, TextEncoder}; - -use crate::extractors::authentication::policies::ActionPolicy; -use crate::extractors::authentication::GuardedData; -use crate::routes::create_all_stats; - -pub async fn get_metrics( - index_scheduler: GuardedData, Data>, - auth_controller: GuardedData, AuthController>, -) -> Result { - let search_rules = &index_scheduler.filters().search_rules; - let response = - create_all_stats((*index_scheduler).clone(), (*auth_controller).clone(), search_rules)?; - - crate::metrics::MEILISEARCH_DB_SIZE_BYTES.set(response.database_size as i64); - crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64); - - for (index, value) in response.indexes.iter() { - crate::metrics::MEILISEARCH_INDEX_DOCS_COUNT - .with_label_values(&[index]) - .set(value.number_of_documents as i64); - } - - let encoder = TextEncoder::new(); - let mut buffer = vec![]; - encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics"); - - let response = String::from_utf8(buffer).expect("Failed to convert bytes to string"); - - Ok(HttpResponse::Ok().insert_header(header::ContentType(mime::TEXT_PLAIN)).body(response)) -} +use prometheus::HistogramTimer; pub struct RouteMetrics; diff --git a/meilisearch/src/option.rs b/meilisearch/src/option.rs index 2a652d7c8..a5d98ab5b 100644 --- a/meilisearch/src/option.rs +++ b/meilisearch/src/option.rs @@ -47,7 +47,6 @@ const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP"; const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS"; const MEILI_DUMP_DIR: &str = "MEILI_DUMP_DIR"; const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL"; -#[cfg(feature = "metrics")] const MEILI_ENABLE_METRICS_ROUTE: &str = "MEILI_ENABLE_METRICS_ROUTE"; const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml"; @@ -288,7 +287,6 @@ pub struct Opt { pub log_level: LogLevel, /// Enables Prometheus metrics and /metrics route. - #[cfg(feature = "metrics")] #[clap(long, env = MEILI_ENABLE_METRICS_ROUTE)] #[serde(default)] pub enable_metrics_route: bool, @@ -384,7 +382,6 @@ impl Opt { config_file_path: _, #[cfg(all(not(debug_assertions), feature = "analytics"))] no_analytics, - #[cfg(feature = "metrics")] enable_metrics_route, } = self; export_to_env_if_not_present(MEILI_DB_PATH, db_path); @@ -423,13 +420,7 @@ impl Opt { export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir); export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level.to_string()); - #[cfg(feature = "metrics")] - { - export_to_env_if_not_present( - MEILI_ENABLE_METRICS_ROUTE, - enable_metrics_route.to_string(), - ); - } + export_to_env_if_not_present(MEILI_ENABLE_METRICS_ROUTE, enable_metrics_route.to_string()); indexer_options.export_to_env(); } diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs new file mode 100644 index 000000000..d0d53e3ec --- /dev/null +++ b/meilisearch/src/routes/metrics.rs @@ -0,0 +1,42 @@ +use actix_web::http::header; +use actix_web::web::{self, Data}; +use actix_web::HttpResponse; +use index_scheduler::IndexScheduler; +use meilisearch_auth::AuthController; +use meilisearch_types::error::ResponseError; +use meilisearch_types::keys::actions; +use prometheus::{Encoder, TextEncoder}; + +use crate::extractors::authentication::policies::ActionPolicy; +use crate::extractors::authentication::GuardedData; +use crate::routes::create_all_stats; + +pub fn configure(config: &mut web::ServiceConfig) { + config.service(web::resource("").route(web::get().to(get_metrics))); +} + +pub async fn get_metrics( + index_scheduler: GuardedData, Data>, + auth_controller: GuardedData, AuthController>, +) -> Result { + let search_rules = &index_scheduler.filters().search_rules; + let response = + create_all_stats((*index_scheduler).clone(), (*auth_controller).clone(), search_rules)?; + + crate::metrics::MEILISEARCH_DB_SIZE_BYTES.set(response.database_size as i64); + crate::metrics::MEILISEARCH_INDEX_COUNT.set(response.indexes.len() as i64); + + for (index, value) in response.indexes.iter() { + crate::metrics::MEILISEARCH_INDEX_DOCS_COUNT + .with_label_values(&[index]) + .set(value.number_of_documents as i64); + } + + let encoder = TextEncoder::new(); + let mut buffer = vec![]; + encoder.encode(&prometheus::gather(), &mut buffer).expect("Failed to encode metrics"); + + let response = String::from_utf8(buffer).expect("Failed to convert bytes to string"); + + Ok(HttpResponse::Ok().insert_header(header::ContentType(mime::TEXT_PLAIN)).body(response)) +} diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index 622e26c75..ce447850d 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -22,11 +22,12 @@ const PAGINATION_DEFAULT_LIMIT: usize = 20; mod api_key; mod dump; pub mod indexes; +mod metrics; mod multi_search; mod swap_indexes; pub mod tasks; -pub fn configure(cfg: &mut web::ServiceConfig) { +pub fn configure(cfg: &mut web::ServiceConfig, enable_metrics: bool) { cfg.service(web::scope("/tasks").configure(tasks::configure)) .service(web::resource("/health").route(web::get().to(get_health))) .service(web::scope("/keys").configure(api_key::configure)) @@ -36,6 +37,10 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::scope("/indexes").configure(indexes::configure)) .service(web::scope("/multi-search").configure(multi_search::configure)) .service(web::scope("/swap-indexes").configure(swap_indexes::configure)); + + if enable_metrics { + cfg.service(web::scope("/metrics").configure(metrics::configure)); + } } #[derive(Debug, Serialize)] diff --git a/meilisearch/tests/auth/authorization.rs b/meilisearch/tests/auth/authorization.rs index b0bc7ab03..2c22a61b3 100644 --- a/meilisearch/tests/auth/authorization.rs +++ b/meilisearch/tests/auth/authorization.rs @@ -52,6 +52,7 @@ pub static AUTHORIZATIONS: Lazy hashset!{"stats.get", "stats.*", "*"}, ("POST", "/dumps") => hashset!{"dumps.create", "dumps.*", "*"}, ("GET", "/version") => hashset!{"version", "*"}, + ("GET", "/metrics") => hashset!{"metrics.get", "metrics.*", "*"}, ("PATCH", "/keys/mykey/") => hashset!{"keys.update", "*"}, ("GET", "/keys/mykey/") => hashset!{"keys.get", "*"}, ("DELETE", "/keys/mykey/") => hashset!{"keys.delete", "*"}, @@ -59,10 +60,6 @@ pub static AUTHORIZATIONS: Lazy hashset!{"keys.get", "*"}, }; - if cfg!(feature = "metrics") { - authorizations.insert(("GET", "/metrics"), hashset! {"metrics.get", "metrics.*", "*"}); - } - authorizations }); diff --git a/meilisearch/tests/common/server.rs b/meilisearch/tests/common/server.rs index 8152edbd0..4b15adbf1 100644 --- a/meilisearch/tests/common/server.rs +++ b/meilisearch/tests/common/server.rs @@ -207,7 +207,6 @@ pub fn default_settings(dir: impl AsRef) -> Opt { max_indexing_memory: MaxMemory::unlimited(), ..Parser::parse_from(None as Option<&str>) }, - #[cfg(feature = "metrics")] enable_metrics_route: true, ..Parser::parse_from(None as Option<&str>) }