From 29ec02d4d42d1d2bd6372df504e3b67b380d7f9d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 26 Jun 2023 12:21:40 +0200 Subject: [PATCH 1/9] Add meilisearch_types::features module --- meilisearch-types/src/features.rs | 13 +++++++++++++ meilisearch-types/src/lib.rs | 1 + 2 files changed, 14 insertions(+) create mode 100644 meilisearch-types/src/features.rs diff --git a/meilisearch-types/src/features.rs b/meilisearch-types/src/features.rs new file mode 100644 index 000000000..6d02fc47b --- /dev/null +++ b/meilisearch-types/src/features.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)] +#[serde(rename_all = "camelCase", default)] +pub struct RuntimeTogglableFeatures { + pub score_details: bool, + pub vector_store: bool, +} + +#[derive(Default, Debug, Clone, Copy)] +pub struct InstanceTogglableFeatures { + pub metrics: bool, +} diff --git a/meilisearch-types/src/lib.rs b/meilisearch-types/src/lib.rs index 99c459903..dbdec14fc 100644 --- a/meilisearch-types/src/lib.rs +++ b/meilisearch-types/src/lib.rs @@ -2,6 +2,7 @@ pub mod compression; pub mod deserr; pub mod document_formats; pub mod error; +pub mod features; pub mod index_uid; pub mod index_uid_pattern; pub mod keys; From 072d81843f4aa9dced5a6d277807de38b8e9eda0 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 22 Jun 2023 22:56:44 +0200 Subject: [PATCH 2/9] Persistently save to DB the status of experimental features --- index-scheduler/src/error.rs | 14 ++++ index-scheduler/src/features.rs | 98 +++++++++++++++++++++++++++ index-scheduler/src/insta_snapshot.rs | 1 + index-scheduler/src/lib.rs | 27 +++++++- meilisearch-types/src/error.rs | 1 + 5 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 index-scheduler/src/features.rs diff --git a/index-scheduler/src/error.rs b/index-scheduler/src/error.rs index acab850d1..ddc6960f7 100644 --- a/index-scheduler/src/error.rs +++ b/index-scheduler/src/error.rs @@ -123,6 +123,8 @@ pub enum Error { IoError(#[from] std::io::Error), #[error(transparent)] Persist(#[from] tempfile::PersistError), + #[error(transparent)] + FeatureNotEnabled(#[from] FeatureNotEnabledError), #[error(transparent)] Anyhow(#[from] anyhow::Error), @@ -142,6 +144,16 @@ pub enum Error { PlannedFailure, } +#[derive(Debug, thiserror::Error)] +#[error( + "{disabled_action} requires enabling the `{feature}` experimental feature. See {issue_link}" +)] +pub struct FeatureNotEnabledError { + pub disabled_action: &'static str, + pub feature: &'static str, + pub issue_link: &'static str, +} + impl Error { pub fn is_recoverable(&self) -> bool { match self { @@ -170,6 +182,7 @@ impl Error { | Error::FileStore(_) | Error::IoError(_) | Error::Persist(_) + | Error::FeatureNotEnabled(_) | Error::Anyhow(_) => true, Error::CreateBatch(_) | Error::CorruptedTaskQueue @@ -214,6 +227,7 @@ impl ErrorCode for Error { Error::FileStore(e) => e.error_code(), Error::IoError(e) => e.error_code(), Error::Persist(e) => e.error_code(), + Error::FeatureNotEnabled(_) => Code::FeatureNotEnabled, // Irrecoverable errors Error::Anyhow(_) => Code::Internal, diff --git a/index-scheduler/src/features.rs b/index-scheduler/src/features.rs new file mode 100644 index 000000000..7f4a30741 --- /dev/null +++ b/index-scheduler/src/features.rs @@ -0,0 +1,98 @@ +use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures}; +use meilisearch_types::heed::types::{SerdeJson, Str}; +use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn}; + +use crate::error::FeatureNotEnabledError; +use crate::Result; + +const EXPERIMENTAL_FEATURES: &str = "experimental-features"; + +#[derive(Clone)] +pub(crate) struct FeatureData { + runtime: Database>, + instance: InstanceTogglableFeatures, +} + +#[derive(Debug, Clone, Copy)] +pub struct RoFeatures { + runtime: RuntimeTogglableFeatures, + instance: InstanceTogglableFeatures, +} + +impl RoFeatures { + fn new(txn: RoTxn<'_>, data: &FeatureData) -> Result { + let runtime = data.runtime_features(txn)?; + Ok(Self { runtime, instance: data.instance }) + } + + pub fn runtime_features(&self) -> RuntimeTogglableFeatures { + self.runtime + } + + pub fn check_score_details(&self) -> Result<()> { + if self.runtime.score_details { + Ok(()) + } else { + Err(FeatureNotEnabledError { + disabled_action: "Computing score details", + feature: "score details", + issue_link: "https://github.com/meilisearch/product/discussions/674", + } + .into()) + } + } + + pub fn check_metrics(&self) -> Result<()> { + if self.instance.metrics { + Ok(()) + } else { + Err(FeatureNotEnabledError { + disabled_action: "Getting metrics", + feature: "metrics", + issue_link: "https://github.com/meilisearch/meilisearch/discussions/3518", + } + .into()) + } + } + + pub fn check_vector(&self) -> Result<()> { + if self.runtime.vector_store { + Ok(()) + } else { + Err(FeatureNotEnabledError { + disabled_action: "Passing `vector` as a query parameter", + feature: "vector store", + issue_link: "https://github.com/meilisearch/meilisearch/discussions/TODO", + } + .into()) + } + } +} + +impl FeatureData { + pub fn new(env: &Env, instance_features: InstanceTogglableFeatures) -> Result { + let mut wtxn = env.write_txn()?; + let runtime_features = env.create_database(&mut wtxn, Some(EXPERIMENTAL_FEATURES))?; + wtxn.commit()?; + + Ok(Self { runtime: runtime_features, instance: instance_features }) + } + + pub fn put_runtime_features( + &self, + mut wtxn: RwTxn, + features: RuntimeTogglableFeatures, + ) -> Result<()> { + self.runtime.put(&mut wtxn, EXPERIMENTAL_FEATURES, &features)?; + wtxn.commit()?; + Ok(()) + } + + fn runtime_features(&self, txn: RoTxn) -> Result { + Ok(self.runtime.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default()) + } + + pub fn features(&self, txn: RoTxn) -> Result { + RoFeatures::new(txn, self) + } +} diff --git a/index-scheduler/src/insta_snapshot.rs b/index-scheduler/src/insta_snapshot.rs index 8369047b0..afcfdb270 100644 --- a/index-scheduler/src/insta_snapshot.rs +++ b/index-scheduler/src/insta_snapshot.rs @@ -28,6 +28,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { started_at, finished_at, index_mapper, + features: _, max_number_of_tasks: _, wake_up: _, dumps_path: _, diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index b86608805..092851edd 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -21,6 +21,7 @@ content of the scheduler or enqueue new tasks. mod autobatcher; mod batch; pub mod error; +mod features; mod index_mapper; #[cfg(test)] mod insta_snapshot; @@ -41,8 +42,10 @@ use std::time::Duration; use dump::{KindDump, TaskDump, UpdateFile}; pub use error::Error; +pub use features::RoFeatures; use file_store::FileStore; use meilisearch_types::error::ResponseError; +use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures}; use meilisearch_types::heed::types::{OwnedType, SerdeBincode, SerdeJson, Str}; use meilisearch_types::heed::{self, Database, Env, RoTxn, RwTxn}; use meilisearch_types::milli::documents::DocumentsBatchBuilder; @@ -247,6 +250,8 @@ pub struct IndexSchedulerOptions { /// The maximum number of tasks stored in the task queue before starting /// to auto schedule task deletions. pub max_number_of_tasks: usize, + /// The experimental features enabled for this instance. + pub instance_features: InstanceTogglableFeatures, } /// Structure which holds meilisearch's indexes and schedules the tasks @@ -290,6 +295,9 @@ pub struct IndexScheduler { /// In charge of creating, opening, storing and returning indexes. pub(crate) index_mapper: IndexMapper, + /// In charge of fetching and setting the status of experimental features. + features: features::FeatureData, + /// Get a signal when a batch needs to be processed. pub(crate) wake_up: Arc, @@ -360,6 +368,7 @@ impl IndexScheduler { planned_failures: self.planned_failures.clone(), #[cfg(test)] run_loop_iteration: self.run_loop_iteration.clone(), + features: self.features.clone(), } } } @@ -398,9 +407,12 @@ impl IndexScheduler { }; let env = heed::EnvOpenOptions::new() - .max_dbs(10) + .max_dbs(11) .map_size(budget.task_db_size) .open(options.tasks_path)?; + + let features = features::FeatureData::new(&env, options.instance_features)?; + let file_store = FileStore::new(&options.update_file_path)?; let mut wtxn = env.write_txn()?; @@ -452,6 +464,7 @@ impl IndexScheduler { planned_failures, #[cfg(test)] run_loop_iteration: Arc::new(RwLock::new(0)), + features, }; this.run(); @@ -1214,6 +1227,17 @@ impl IndexScheduler { Ok(IndexStats { is_indexing, inner_stats: index_stats }) } + pub fn features(&self) -> Result { + let rtxn = self.read_txn()?; + self.features.features(rtxn) + } + + pub fn put_runtime_features(&self, features: RuntimeTogglableFeatures) -> Result<()> { + let wtxn = self.env.write_txn().map_err(Error::HeedTransaction)?; + self.features.put_runtime_features(wtxn, features)?; + Ok(()) + } + pub(crate) fn delete_persisted_task_data(&self, task: &Task) -> Result<()> { match task.content_uuid() { Some(content_file) => self.delete_update_file(content_file), @@ -1534,6 +1558,7 @@ mod tests { indexer_config, autobatching_enabled: true, max_number_of_tasks: 1_000_000, + instance_features: Default::default(), }; configuration(&mut options); diff --git a/meilisearch-types/src/error.rs b/meilisearch-types/src/error.rs index 3e08498de..6d81ff241 100644 --- a/meilisearch-types/src/error.rs +++ b/meilisearch-types/src/error.rs @@ -271,6 +271,7 @@ InvalidTaskStatuses , InvalidRequest , BAD_REQUEST ; InvalidTaskTypes , InvalidRequest , BAD_REQUEST ; InvalidTaskUids , InvalidRequest , BAD_REQUEST ; IoError , System , UNPROCESSABLE_ENTITY; +FeatureNotEnabled , InvalidRequest , BAD_REQUEST ; MalformedPayload , InvalidRequest , BAD_REQUEST ; MaxFieldsLimitExceeded , InvalidRequest , BAD_REQUEST ; MissingApiKeyActions , InvalidRequest , BAD_REQUEST ; From dac77dfd14f98a0d180f8bf950236e41e2d666f8 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 22 Jun 2023 22:57:59 +0200 Subject: [PATCH 3/9] Add new permissions for experimental-features route --- meilisearch-types/src/keys.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/meilisearch-types/src/keys.rs b/meilisearch-types/src/keys.rs index b2389b238..2a912aee6 100644 --- a/meilisearch-types/src/keys.rs +++ b/meilisearch-types/src/keys.rs @@ -274,6 +274,12 @@ pub enum Action { #[serde(rename = "keys.delete")] #[deserr(rename = "keys.delete")] KeysDelete, + #[serde(rename = "experimental.get")] + #[deserr(rename = "experimental.get")] + ExperimentalFeaturesGet, + #[serde(rename = "experimental.update")] + #[deserr(rename = "experimental.update")] + ExperimentalFeaturesUpdate, } impl Action { @@ -310,6 +316,8 @@ impl Action { KEYS_GET => Some(Self::KeysGet), KEYS_UPDATE => Some(Self::KeysUpdate), KEYS_DELETE => Some(Self::KeysDelete), + EXPERIMENTAL_FEATURES_GET => Some(Self::ExperimentalFeaturesGet), + EXPERIMENTAL_FEATURES_UPDATE => Some(Self::ExperimentalFeaturesUpdate), _otherwise => None, } } @@ -352,4 +360,6 @@ pub mod actions { pub const KEYS_GET: u8 = KeysGet.repr(); pub const KEYS_UPDATE: u8 = KeysUpdate.repr(); pub const KEYS_DELETE: u8 = KeysDelete.repr(); + pub const EXPERIMENTAL_FEATURES_GET: u8 = ExperimentalFeaturesGet.repr(); + pub const EXPERIMENTAL_FEATURES_UPDATE: u8 = ExperimentalFeaturesUpdate.repr(); } From eef9293630aed63c3e1cf0aa1662d5323d0f42d0 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 22 Jun 2023 22:58:54 +0200 Subject: [PATCH 4/9] New route to set some experimental features --- meilisearch/src/routes/features.rs | 102 +++++++++++++++++++++++++++++ meilisearch/src/routes/mod.rs | 4 +- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 meilisearch/src/routes/features.rs diff --git a/meilisearch/src/routes/features.rs b/meilisearch/src/routes/features.rs new file mode 100644 index 000000000..08a0f5ba1 --- /dev/null +++ b/meilisearch/src/routes/features.rs @@ -0,0 +1,102 @@ +use actix_web::web::{self, Data}; +use actix_web::{HttpRequest, HttpResponse}; +use deserr::actix_web::AwebJson; +use deserr::Deserr; +use index_scheduler::IndexScheduler; +use log::debug; +use meilisearch_types::deserr::DeserrJsonError; +use meilisearch_types::error::ResponseError; +use meilisearch_types::keys::actions; +use serde_json::json; + +use crate::analytics::Analytics; +use crate::extractors::authentication::policies::ActionPolicy; +use crate::extractors::authentication::GuardedData; +use crate::extractors::sequential_extractor::SeqHandler; + +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service( + web::resource("") + .route(web::get().to(SeqHandler(get_features))) + .route(web::patch().to(SeqHandler(patch_features))) + .route(web::delete().to(SeqHandler(delete_features))) + .route(web::post().to(SeqHandler(post_features))), + ); +} + +async fn get_features( + index_scheduler: GuardedData< + ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_GET }>, + Data, + >, + req: HttpRequest, + analytics: Data, +) -> Result { + let features = index_scheduler.features()?; + + analytics.publish("Experimental features Seen".to_string(), json!(null), Some(&req)); + debug!("returns: {:?}", features.runtime_features()); + Ok(HttpResponse::Ok().json(features.runtime_features())) +} + +#[derive(Debug, Deserr)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +pub struct RuntimeTogglableFeatures { + #[deserr(default)] + pub score_details: Option, + #[deserr(default)] + pub vector_store: Option, +} + +async fn patch_features( + index_scheduler: GuardedData< + ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_UPDATE }>, + Data, + >, + new_features: AwebJson, + analytics: Data, +) -> Result { + let features = index_scheduler.features()?; + + let old_features = features.runtime_features(); + + let new_features = meilisearch_types::features::RuntimeTogglableFeatures { + score_details: new_features.0.score_details.unwrap_or(old_features.score_details), + vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store), + }; + + analytics.publish("Experimental features PATCH".to_string(), json!(new_features), None); + index_scheduler.put_runtime_features(new_features)?; + Ok(HttpResponse::Ok().json(new_features)) +} + +async fn post_features( + index_scheduler: GuardedData< + ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_UPDATE }>, + Data, + >, + new_features: AwebJson, + analytics: Data, +) -> Result { + let new_features = meilisearch_types::features::RuntimeTogglableFeatures { + score_details: new_features.0.score_details.unwrap_or(false), + vector_store: new_features.0.vector_store.unwrap_or(false), + }; + + analytics.publish("Experimental features POST".to_string(), json!(new_features), None); + index_scheduler.put_runtime_features(new_features)?; + Ok(HttpResponse::Ok().json(new_features)) +} + +async fn delete_features( + index_scheduler: GuardedData< + ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_UPDATE }>, + Data, + >, + analytics: Data, +) -> Result { + let deleted_features = Default::default(); + analytics.publish("Experimental features DELETE".to_string(), json!(null), None); + index_scheduler.put_runtime_features(deleted_features)?; + Ok(HttpResponse::Ok().json(deleted_features)) +} diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index 57d670b5f..9c5d29119 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -20,6 +20,7 @@ const PAGINATION_DEFAULT_LIMIT: usize = 20; mod api_key; mod dump; +pub mod features; pub mod indexes; mod metrics; mod multi_search; @@ -35,7 +36,8 @@ pub fn configure(cfg: &mut web::ServiceConfig, enable_metrics: bool) { .service(web::resource("/version").route(web::get().to(get_version))) .service(web::scope("/indexes").configure(indexes::configure)) .service(web::scope("/multi-search").configure(multi_search::configure)) - .service(web::scope("/swap-indexes").configure(swap_indexes::configure)); + .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) + .service(web::scope("/experimental-features").configure(features::configure)); if enable_metrics { cfg.service(web::scope("/metrics").configure(metrics::configure)); From bb6448dc2ea7fa22998e95cf3a3e9454eaa43063 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 22 Jun 2023 23:10:22 +0200 Subject: [PATCH 5/9] Compute instance features from CLI options --- meilisearch/src/lib.rs | 2 ++ meilisearch/src/option.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index bee53f6f8..a9bd68934 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -221,6 +221,7 @@ fn open_or_create_database_unchecked( // we don't want to create anything in the data.ms yet, thus we // wrap our two builders in a closure that'll be executed later. let auth_controller = AuthController::new(&opt.db_path, &opt.master_key); + let instance_features = opt.to_instance_features(); let index_scheduler_builder = || -> anyhow::Result<_> { Ok(IndexScheduler::new(IndexSchedulerOptions { version_file_path: opt.db_path.join(VERSION_FILE_NAME), @@ -238,6 +239,7 @@ fn open_or_create_database_unchecked( max_number_of_tasks: 1_000_000, index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize, index_count: DEFAULT_INDEX_COUNT, + instance_features, })?) }; diff --git a/meilisearch/src/option.rs b/meilisearch/src/option.rs index 0511b5033..eaf96ee0c 100644 --- a/meilisearch/src/option.rs +++ b/meilisearch/src/option.rs @@ -12,6 +12,7 @@ use std::{env, fmt, fs}; use byte_unit::{Byte, ByteError}; use clap::Parser; +use meilisearch_types::features::InstanceTogglableFeatures; use meilisearch_types::milli::update::IndexerConfig; use rustls::server::{ AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, ServerSessionMemoryCache, @@ -486,6 +487,10 @@ impl Opt { Ok(None) } } + + pub(crate) fn to_instance_features(&self) -> InstanceTogglableFeatures { + InstanceTogglableFeatures { metrics: self.experimental_enable_metrics } + } } #[derive(Debug, Default, Clone, Parser, Deserialize)] From 6196a536682f39f6e1a5c0f1a6d4edba0365babf Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 22 Jun 2023 23:12:29 +0200 Subject: [PATCH 6/9] Gate score_details behind a runtime experimental feature flag --- meilisearch/src/routes/indexes/search.rs | 9 +++++++-- meilisearch/src/routes/multi_search.rs | 4 +++- meilisearch/src/search.rs | 6 ++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/meilisearch/src/routes/indexes/search.rs b/meilisearch/src/routes/indexes/search.rs index cb70147cd..3ab093b5d 100644 --- a/meilisearch/src/routes/indexes/search.rs +++ b/meilisearch/src/routes/indexes/search.rs @@ -151,7 +151,9 @@ pub async fn search_with_url_query( let mut aggregate = SearchAggregator::from_query(&query, &req); let index = index_scheduler.index(&index_uid)?; - let search_result = tokio::task::spawn_blocking(move || perform_search(&index, query)).await?; + let features = index_scheduler.features()?; + let search_result = + tokio::task::spawn_blocking(move || perform_search(&index, query, features)).await?; if let Ok(ref search_result) = search_result { aggregate.succeed(search_result); } @@ -183,7 +185,10 @@ pub async fn search_with_post( let mut aggregate = SearchAggregator::from_query(&query, &req); let index = index_scheduler.index(&index_uid)?; - let search_result = tokio::task::spawn_blocking(move || perform_search(&index, query)).await?; + + let features = index_scheduler.features()?; + let search_result = + tokio::task::spawn_blocking(move || perform_search(&index, query, features)).await?; if let Ok(ref search_result) = search_result { aggregate.succeed(search_result); } diff --git a/meilisearch/src/routes/multi_search.rs b/meilisearch/src/routes/multi_search.rs index fd78df5e5..e257af1cf 100644 --- a/meilisearch/src/routes/multi_search.rs +++ b/meilisearch/src/routes/multi_search.rs @@ -41,6 +41,7 @@ pub async fn multi_search_with_post( let queries = params.into_inner().queries; let mut multi_aggregate = MultiSearchAggregator::from_queries(&queries, &req); + let features = index_scheduler.features()?; // Explicitly expect a `(ResponseError, usize)` for the error type rather than `ResponseError` only, // so that `?` doesn't work if it doesn't use `with_index`, ensuring that it is not forgotten in case of code @@ -74,8 +75,9 @@ pub async fn multi_search_with_post( err }) .with_index(query_index)?; + let search_result = - tokio::task::spawn_blocking(move || perform_search(&index, query)) + tokio::task::spawn_blocking(move || perform_search(&index, query, features)) .await .with_index(query_index)?; diff --git a/meilisearch/src/search.rs b/meilisearch/src/search.rs index b2858ead3..62f49c148 100644 --- a/meilisearch/src/search.rs +++ b/meilisearch/src/search.rs @@ -5,6 +5,7 @@ use std::time::Instant; use deserr::Deserr; use either::Either; +use index_scheduler::RoFeatures; use meilisearch_auth::IndexSearchRules; use meilisearch_types::deserr::DeserrJsonError; use meilisearch_types::error::deserr_codes::*; @@ -281,6 +282,7 @@ pub fn add_search_rules(query: &mut SearchQuery, rules: IndexSearchRules) { pub fn perform_search( index: &Index, query: SearchQuery, + features: RoFeatures, ) -> Result { let before_search = Instant::now(); let rtxn = index.read_txn()?; @@ -306,6 +308,10 @@ pub fn perform_search( ScoringStrategy::Skip }); + if query.show_ranking_score_details { + features.check_score_details()?; + } + // compute the offset on the limit depending on the pagination mode. let (offset, limit) = if is_finite_pagination { let limit = query.hits_per_page.unwrap_or_else(DEFAULT_SEARCH_LIMIT); From cca6e47ec1e65cdf78d30b1f6ca6e2b626e3a078 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 22 Jun 2023 23:14:01 +0200 Subject: [PATCH 7/9] Errors when GETting metrics without the feature gate --- meilisearch/src/lib.rs | 2 +- meilisearch/src/routes/metrics.rs | 1 + meilisearch/src/routes/mod.rs | 7 ++----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index a9bd68934..fc8017715 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -111,7 +111,7 @@ pub fn create_app( analytics.clone(), ) }) - .configure(|cfg| routes::configure(cfg, opt.experimental_enable_metrics)) + .configure(routes::configure) .configure(|s| dashboard(s, enable_dashboard)); let app = app.wrap(actix_web::middleware::Condition::new( diff --git a/meilisearch/src/routes/metrics.rs b/meilisearch/src/routes/metrics.rs index a7d41e33e..492fec417 100644 --- a/meilisearch/src/routes/metrics.rs +++ b/meilisearch/src/routes/metrics.rs @@ -19,6 +19,7 @@ pub async fn get_metrics( index_scheduler: GuardedData, Data>, auth_controller: Data, ) -> Result { + index_scheduler.features()?.check_metrics()?; let auth_filters = index_scheduler.filters(); if !auth_filters.all_indexes_authorized() { let mut error = ResponseError::from(AuthenticationError::InvalidToken); diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index 9c5d29119..d14ddf48a 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -27,7 +27,7 @@ mod multi_search; mod swap_indexes; pub mod tasks; -pub fn configure(cfg: &mut web::ServiceConfig, enable_metrics: bool) { +pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service(web::scope("/tasks").configure(tasks::configure)) .service(web::resource("/health").route(web::get().to(get_health))) .service(web::scope("/keys").configure(api_key::configure)) @@ -37,11 +37,8 @@ pub fn configure(cfg: &mut web::ServiceConfig, enable_metrics: bool) { .service(web::scope("/indexes").configure(indexes::configure)) .service(web::scope("/multi-search").configure(multi_search::configure)) .service(web::scope("/swap-indexes").configure(swap_indexes::configure)) + .service(web::scope("/metrics").configure(metrics::configure)) .service(web::scope("/experimental-features").configure(features::configure)); - - if enable_metrics { - cfg.service(web::scope("/metrics").configure(metrics::configure)); - } } #[derive(Debug, Serialize)] From 5a83cecb0f47b768f2a03a3885033ef8bc72e419 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 22 Jun 2023 23:34:57 +0200 Subject: [PATCH 8/9] fix tests --- meilisearch/tests/auth/api_keys.rs | 2 +- meilisearch/tests/auth/errors.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch/tests/auth/api_keys.rs b/meilisearch/tests/auth/api_keys.rs index c3916cd03..a356a4dd6 100644 --- a/meilisearch/tests/auth/api_keys.rs +++ b/meilisearch/tests/auth/api_keys.rs @@ -422,7 +422,7 @@ async fn error_add_api_key_invalid_parameters_actions() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r###" { - "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`", + "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" diff --git a/meilisearch/tests/auth/errors.rs b/meilisearch/tests/auth/errors.rs index cd1e9acf2..f81574abd 100644 --- a/meilisearch/tests/auth/errors.rs +++ b/meilisearch/tests/auth/errors.rs @@ -90,7 +90,7 @@ async fn create_api_key_bad_actions() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`", + "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" From 13e9b4c2e52a6fc63cb062a3cced3ff7d2f8d5e0 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 26 Jun 2023 12:24:55 +0200 Subject: [PATCH 9/9] Add dump support --- dump/src/lib.rs | 2 ++ dump/src/reader/compat/v5_to_v6.rs | 4 ++++ dump/src/reader/mod.rs | 9 +++++++++ dump/src/reader/v6/mod.rs | 25 +++++++++++++++++++++++++ dump/src/writer.rs | 8 ++++++++ index-scheduler/src/batch.rs | 4 ++++ meilisearch/src/lib.rs | 18 +++++++++++------- 7 files changed, 63 insertions(+), 7 deletions(-) diff --git a/dump/src/lib.rs b/dump/src/lib.rs index ed76d708e..5ae9ddfed 100644 --- a/dump/src/lib.rs +++ b/dump/src/lib.rs @@ -412,6 +412,8 @@ pub(crate) mod test { } keys.flush().unwrap(); + // ========== TODO: create features here + // create the dump let mut file = tempfile::tempfile().unwrap(); dump.persist_to(&mut file).unwrap(); diff --git a/dump/src/reader/compat/v5_to_v6.rs b/dump/src/reader/compat/v5_to_v6.rs index c98cc819b..b4c851d28 100644 --- a/dump/src/reader/compat/v5_to_v6.rs +++ b/dump/src/reader/compat/v5_to_v6.rs @@ -191,6 +191,10 @@ impl CompatV5ToV6 { }) }))) } + + pub fn features(&self) -> Result> { + Ok(None) + } } pub enum CompatIndexV5ToV6 { diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index a5a66591b..0899579f8 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -107,6 +107,13 @@ impl DumpReader { DumpReader::Compat(compat) => compat.keys(), } } + + pub fn features(&self) -> Result> { + match self { + DumpReader::Current(current) => Ok(current.features()), + DumpReader::Compat(compat) => compat.features(), + } + } } impl From for DumpReader { @@ -189,6 +196,8 @@ pub(crate) mod test { use super::*; + // TODO: add `features` to tests + #[test] fn import_dump_v5() { let dump = File::open("tests/assets/v5.dump").unwrap(); diff --git a/dump/src/reader/v6/mod.rs b/dump/src/reader/v6/mod.rs index f0ad81116..4e980e03e 100644 --- a/dump/src/reader/v6/mod.rs +++ b/dump/src/reader/v6/mod.rs @@ -2,6 +2,7 @@ use std::fs::{self, File}; use std::io::{BufRead, BufReader, ErrorKind}; use std::path::Path; +use log::debug; pub use meilisearch_types::milli; use tempfile::TempDir; use time::OffsetDateTime; @@ -18,6 +19,7 @@ pub type Unchecked = meilisearch_types::settings::Unchecked; pub type Task = crate::TaskDump; pub type Key = meilisearch_types::keys::Key; +pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; // ===== Other types to clarify the code of the compat module // everything related to the tasks @@ -47,6 +49,7 @@ pub struct V6Reader { metadata: Metadata, tasks: BufReader, keys: BufReader, + features: Option, } impl V6Reader { @@ -58,11 +61,29 @@ impl V6Reader { Err(e) => return Err(e.into()), }; + let feature_file = match fs::read(dump.path().join("experimental-features.json")) { + Ok(feature_file) => Some(feature_file), + Err(error) => match error.kind() { + // Allows the file to be missing, this will only result in all experimental features disabled. + ErrorKind::NotFound => { + debug!("`experimental-features.json` not found in dump"); + None + } + _ => return Err(error.into()), + }, + }; + let features = if let Some(feature_file) = feature_file { + Some(serde_json::from_reader(&*feature_file)?) + } else { + None + }; + Ok(V6Reader { metadata: serde_json::from_reader(&*meta_file)?, instance_uid, tasks: BufReader::new(File::open(dump.path().join("tasks").join("queue.jsonl"))?), keys: BufReader::new(File::open(dump.path().join("keys.jsonl"))?), + features, dump, }) } @@ -129,6 +150,10 @@ impl V6Reader { (&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }), ) } + + pub fn features(&self) -> Option { + self.features + } } pub struct UpdateFile { diff --git a/dump/src/writer.rs b/dump/src/writer.rs index 9776dd225..695610f78 100644 --- a/dump/src/writer.rs +++ b/dump/src/writer.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use flate2::write::GzEncoder; use flate2::Compression; +use meilisearch_types::features::RuntimeTogglableFeatures; use meilisearch_types::keys::Key; use meilisearch_types::settings::{Checked, Settings}; use serde_json::{Map, Value}; @@ -53,6 +54,13 @@ impl DumpWriter { TaskWriter::new(self.dir.path().join("tasks")) } + pub fn create_experimental_features(&self, features: RuntimeTogglableFeatures) -> Result<()> { + Ok(std::fs::write( + self.dir.path().join("experimental-features.json"), + serde_json::to_string(&features)?, + )?) + } + pub fn persist_to(self, mut writer: impl Write) -> Result<()> { let gz_encoder = GzEncoder::new(&mut writer, Compression::default()); let mut tar_encoder = tar::Builder::new(gz_encoder); diff --git a/index-scheduler/src/batch.rs b/index-scheduler/src/batch.rs index 985dad976..2948e7506 100644 --- a/index-scheduler/src/batch.rs +++ b/index-scheduler/src/batch.rs @@ -839,6 +839,10 @@ impl IndexScheduler { Ok(()) })?; + // 4. Dump experimental feature settings + let features = self.features()?.runtime_features(); + dump.create_experimental_features(features)?; + let dump_uid = started_at.format(format_description!( "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" )).unwrap(); diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index fc8017715..ef821f49f 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -309,12 +309,16 @@ fn import_dump( keys.push(key); } + // 3. Import the runtime features. + let features = dump_reader.features()?.unwrap_or_default(); + index_scheduler.put_runtime_features(features)?; + let indexer_config = index_scheduler.indexer_config(); // /!\ The tasks must be imported AFTER importing the indexes or else the scheduler might // try to process tasks while we're trying to import the indexes. - // 3. Import the indexes. + // 4. Import the indexes. for index_reader in dump_reader.indexes()? { let mut index_reader = index_reader?; let metadata = index_reader.metadata(); @@ -326,19 +330,19 @@ fn import_dump( let mut wtxn = index.write_txn()?; let mut builder = milli::update::Settings::new(&mut wtxn, &index, indexer_config); - // 3.1 Import the primary key if there is one. + // 4.1 Import the primary key if there is one. if let Some(ref primary_key) = metadata.primary_key { builder.set_primary_key(primary_key.to_string()); } - // 3.2 Import the settings. + // 4.2 Import the settings. log::info!("Importing the settings."); let settings = index_reader.settings()?; apply_settings_to_builder(&settings, &mut builder); builder.execute(|indexing_step| log::debug!("update: {:?}", indexing_step), || false)?; - // 3.3 Import the documents. - // 3.3.1 We need to recreate the grenad+obkv format accepted by the index. + // 4.3 Import the documents. + // 4.3.1 We need to recreate the grenad+obkv format accepted by the index. log::info!("Importing the documents."); let file = tempfile::tempfile()?; let mut builder = DocumentsBatchBuilder::new(BufWriter::new(file)); @@ -349,7 +353,7 @@ fn import_dump( // This flush the content of the batch builder. let file = builder.into_inner()?.into_inner()?; - // 3.3.2 We feed it to the milli index. + // 4.3.2 We feed it to the milli index. let reader = BufReader::new(file); let reader = DocumentsBatchReader::from_reader(reader)?; @@ -374,7 +378,7 @@ fn import_dump( let mut index_scheduler_dump = index_scheduler.register_dumped_task()?; - // 4. Import the tasks. + // 5. Import the tasks. for ret in dump_reader.tasks()? { let (task, file) = ret?; index_scheduler_dump.register_dumped_task(task, file)?;