From 403226a02929e44ef4a913a68359f62fb9fa7326 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Tue, 6 Sep 2022 09:23:16 +0200 Subject: [PATCH 01/15] Add support for config file --- Cargo.lock | 1 + meilisearch-http/Cargo.toml | 1 + meilisearch-http/src/main.rs | 5 +- meilisearch-http/src/option.rs | 172 ++++++++++++++++++++++++++------- meilisearch-lib/src/lib.rs | 13 +++ meilisearch-lib/src/options.rs | 50 ++++++++-- 6 files changed, 196 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index babbd0ab2..18375d2fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2087,6 +2087,7 @@ dependencies = [ "time 0.3.9", "tokio", "tokio-stream", + "toml", "urlencoding", "uuid", "vergen", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 38f9a83fc..88be3c154 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -76,6 +76,7 @@ thiserror = "1.0.30" time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } tokio = { version = "1.17.0", features = ["full"] } tokio-stream = "0.1.8" +toml = "0.5.9" uuid = { version = "1.1.2", features = ["serde", "v4"] } walkdir = "2.3.2" prometheus = { version = "0.13.0", features = ["process"], optional = true } diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 9627aeef8..e74c1e056 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use actix_web::http::KeepAlive; use actix_web::HttpServer; -use clap::Parser; use meilisearch_auth::AuthController; use meilisearch_http::analytics; use meilisearch_http::analytics::Analytics; @@ -29,7 +28,9 @@ fn setup(opt: &Opt) -> anyhow::Result<()> { #[actix_web::main] async fn main() -> anyhow::Result<()> { - let opt = Opt::parse(); + let opt = Opt::build(); + + println!("{:?}", opt); setup(&opt)?; diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index bdfa283a6..143dfb231 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -4,8 +4,11 @@ use std::path::PathBuf; use std::sync::Arc; use byte_unit::Byte; -use clap::Parser; -use meilisearch_lib::options::{IndexerOpts, SchedulerConfig}; +use clap::{Arg, Command, Parser}; +use meilisearch_lib::{ + export_to_env_if_not_present, + options::{IndexerOpts, SchedulerConfig}, +}; use rustls::{ server::{ AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, @@ -14,90 +17,114 @@ use rustls::{ RootCertStore, }; use rustls_pemfile::{certs, pkcs8_private_keys, rsa_private_keys}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; -#[derive(Debug, Clone, Parser, Serialize)] +const MEILI_DB_PATH: &str = "MEILI_DB_PATH"; +const MEILI_HTTP_ADDR: &str = "MEILI_HTTP_ADDR"; +const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY"; +const MEILI_ENV: &str = "MEILI_ENV"; +#[cfg(all(not(debug_assertions), feature = "analytics"))] +const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS"; +const MEILI_MAX_INDEX_SIZE: &str = "MEILI_MAX_INDEX_SIZE"; +const MEILI_MAX_TASK_DB_SIZE: &str = "MEILI_MAX_TASK_DB_SIZE"; +const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT"; +const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH"; +const MEILI_SSL_KEY_PATH: &str = "MEILI_SSL_KEY_PATH"; +const MEILI_SSL_AUTH_PATH: &str = "MEILI_SSL_AUTH_PATH"; +const MEILI_SSL_OCSP_PATH: &str = "MEILI_SSL_OCSP_PATH"; +const MEILI_SSL_REQUIRE_AUTH: &str = "MEILI_SSL_REQUIRE_AUTH"; +const MEILI_SSL_RESUMPTION: &str = "MEILI_SSL_RESUMPTION"; +const MEILI_SSL_TICKETS: &str = "MEILI_SSL_TICKETS"; +const MEILI_SNAPSHOT_DIR: &str = "MEILI_SNAPSHOT_DIR"; +const MEILI_SCHEDULE_SNAPSHOT: &str = "MEILI_SCHEDULE_SNAPSHOT"; +const MEILI_SNAPSHOT_INTERVAL_SEC: &str = "MEILI_SNAPSHOT_INTERVAL_SEC"; +const MEILI_DUMPS_DIR: &str = "MEILI_DUMPS_DIR"; +const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL"; +#[cfg(feature = "metrics")] +const MEILI_ENABLE_METRICS_ROUTE: &str = "MEILI_ENABLE_METRICS_ROUTE"; + +#[derive(Debug, Clone, Parser, Serialize, Deserialize)] #[clap(version)] pub struct Opt { /// The destination where the database must be created. - #[clap(long, env = "MEILI_DB_PATH", default_value = "./data.ms")] + #[clap(long, env = MEILI_DB_PATH, default_value = "./data.ms")] pub db_path: PathBuf, /// The address on which the http server will listen. - #[clap(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")] + #[clap(long, env = MEILI_HTTP_ADDR, default_value = "127.0.0.1:7700")] pub http_addr: String, /// The master key allowing you to do everything on the server. - #[serde(skip)] - #[clap(long, env = "MEILI_MASTER_KEY")] + #[serde(skip_serializing)] + #[clap(long, env = MEILI_MASTER_KEY)] pub master_key: Option, /// This environment variable must be set to `production` if you are running in production. /// If the server is running in development mode more logs will be displayed, /// and the master key can be avoided which implies that there is no security on the updates routes. /// This is useful to debug when integrating the engine with another service. - #[clap(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)] + #[clap(long, env = MEILI_ENV, default_value = "development", possible_values = &POSSIBLE_ENV)] pub env: String, /// Do not send analytics to Meili. #[cfg(all(not(debug_assertions), feature = "analytics"))] - #[serde(skip)] // we can't send true - #[clap(long, env = "MEILI_NO_ANALYTICS")] + #[serde(skip_serializing)] // we can't send true + #[clap(long, env = MEILI_NO_ANALYTICS)] pub no_analytics: bool, /// The maximum size, in bytes, of the main lmdb database directory - #[clap(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")] + #[clap(long, env = MEILI_MAX_INDEX_SIZE, default_value = "100 GiB")] pub max_index_size: Byte, /// The maximum size, in bytes, of the update lmdb database directory - #[clap(long, env = "MEILI_MAX_TASK_DB_SIZE", default_value = "100 GiB")] + #[clap(long, env = MEILI_MAX_TASK_DB_SIZE, default_value = "100 GiB")] pub max_task_db_size: Byte, /// The maximum size, in bytes, of accepted JSON payloads - #[clap(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")] + #[clap(long, env = MEILI_HTTP_PAYLOAD_SIZE_LIMIT, default_value = "100 MB")] pub http_payload_size_limit: Byte, /// Read server certificates from CERTFILE. /// This should contain PEM-format certificates /// in the right order (the first certificate should /// certify KEYFILE, the last should be a root CA). - #[serde(skip)] - #[clap(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))] + #[serde(skip_serializing)] + #[clap(long, env = MEILI_SSL_CERT_PATH, parse(from_os_str))] pub ssl_cert_path: Option, /// Read private key from KEYFILE. This should be a RSA /// private key or PKCS8-encoded private key, in PEM format. - #[serde(skip)] - #[clap(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))] + #[serde(skip_serializing)] + #[clap(long, env = MEILI_SSL_KEY_PATH, parse(from_os_str))] pub ssl_key_path: Option, /// Enable client authentication, and accept certificates /// signed by those roots provided in CERTFILE. - #[clap(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))] - #[serde(skip)] + #[clap(long, env = MEILI_SSL_AUTH_PATH, parse(from_os_str))] + #[serde(skip_serializing)] pub ssl_auth_path: Option, /// Read DER-encoded OCSP response from OCSPFILE and staple to certificate. /// Optional - #[serde(skip)] - #[clap(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))] + #[serde(skip_serializing)] + #[clap(long, env = MEILI_SSL_OCSP_PATH, parse(from_os_str))] pub ssl_ocsp_path: Option, /// Send a fatal alert if the client does not complete client authentication. - #[serde(skip)] - #[clap(long, env = "MEILI_SSL_REQUIRE_AUTH")] + #[serde(skip_serializing)] + #[clap(long, env = MEILI_SSL_REQUIRE_AUTH)] pub ssl_require_auth: bool, /// SSL support session resumption - #[serde(skip)] - #[clap(long, env = "MEILI_SSL_RESUMPTION")] + #[serde(skip_serializing)] + #[clap(long, env = MEILI_SSL_RESUMPTION)] pub ssl_resumption: bool, /// SSL support tickets. - #[serde(skip)] - #[clap(long, env = "MEILI_SSL_TICKETS")] + #[serde(skip_serializing)] + #[clap(long, env = MEILI_SSL_TICKETS)] pub ssl_tickets: bool, /// Defines the path of the snapshot file to import. @@ -123,15 +150,15 @@ pub struct Opt { pub ignore_snapshot_if_db_exists: bool, /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap. - #[clap(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")] + #[clap(long, env = MEILI_SNAPSHOT_DIR, default_value = "snapshots/")] pub snapshot_dir: PathBuf, /// Activate snapshot scheduling. - #[clap(long, env = "MEILI_SCHEDULE_SNAPSHOT")] + #[clap(long, env = MEILI_SCHEDULE_SNAPSHOT)] pub schedule_snapshot: bool, /// Defines time interval, in seconds, between each snapshot creation. - #[clap(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h + #[clap(long, env = MEILI_SNAPSHOT_INTERVAL_SEC, default_value = "86400")] // 24h pub snapshot_interval_sec: u64, /// Import a dump from the specified path, must be a `.dump` file. @@ -147,16 +174,16 @@ pub struct Opt { pub ignore_dump_if_db_exists: bool, /// Folder where dumps are created when the dump route is called. - #[clap(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")] + #[clap(long, env = MEILI_DUMPS_DIR, default_value = "dumps/")] pub dumps_dir: PathBuf, /// Set the log level - #[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")] + #[clap(long, env = MEILI_LOG_LEVEL, default_value = "info")] pub log_level: String, /// Enables Prometheus metrics and /metrics route. #[cfg(feature = "metrics")] - #[clap(long, env = "MEILI_ENABLE_METRICS_ROUTE")] + #[clap(long, env = MEILI_ENABLE_METRICS_ROUTE)] pub enable_metrics_route: bool, #[serde(flatten)] @@ -175,6 +202,83 @@ impl Opt { !self.no_analytics } + pub fn build() -> Self { + let args = Command::new("config") + .arg( + Arg::new("config_file_path") + .long("config-file-path") + .takes_value(true) + .default_value("./config.toml") + .help("Path to a config file, must be TOML format"), + ) + .get_matches(); + let config_file_path = args + .value_of("config_file_path") + .expect("default value present"); + if let Some(Ok(opts_from_file)) = match std::fs::read_to_string(config_file_path) { + Ok(config_str) => Some(toml::from_str::(&config_str)), + Err(err) => { + log::debug!("can't read {} : {}", config_file_path, err); + None + } + } { + opts_from_file.export_to_env(); + } + + Opt::parse() + } + + fn export_to_env(self) { + export_to_env_if_not_present(MEILI_DB_PATH, self.db_path); + export_to_env_if_not_present(MEILI_HTTP_ADDR, self.http_addr); + if let Some(master_key) = self.master_key { + export_to_env_if_not_present(MEILI_MASTER_KEY, master_key); + } + export_to_env_if_not_present(MEILI_ENV, self.env); + #[cfg(all(not(debug_assertions), feature = "analytics"))] + { + export_to_env_if_not_present(MEILI_NO_ANALYTICS, self.no_analytics); + } + export_to_env_if_not_present(MEILI_MAX_INDEX_SIZE, self.max_index_size.to_string()); + export_to_env_if_not_present(MEILI_MAX_TASK_DB_SIZE, self.max_task_db_size.to_string()); + export_to_env_if_not_present( + MEILI_HTTP_PAYLOAD_SIZE_LIMIT, + self.http_payload_size_limit.to_string(), + ); + if let Some(ssl_cert_path) = self.ssl_cert_path { + export_to_env_if_not_present(MEILI_SSL_CERT_PATH, ssl_cert_path); + } + if let Some(ssl_key_path) = self.ssl_key_path { + export_to_env_if_not_present(MEILI_SSL_KEY_PATH, ssl_key_path); + } + if let Some(ssl_auth_path) = self.ssl_auth_path { + export_to_env_if_not_present(MEILI_SSL_AUTH_PATH, ssl_auth_path); + } + if let Some(ssl_ocsp_path) = self.ssl_ocsp_path { + export_to_env_if_not_present(MEILI_SSL_OCSP_PATH, ssl_ocsp_path); + } + export_to_env_if_not_present(MEILI_SSL_REQUIRE_AUTH, self.ssl_require_auth.to_string()); + export_to_env_if_not_present(MEILI_SSL_RESUMPTION, self.ssl_resumption.to_string()); + export_to_env_if_not_present(MEILI_SSL_TICKETS, self.ssl_tickets.to_string()); + export_to_env_if_not_present(MEILI_SNAPSHOT_DIR, self.snapshot_dir); + export_to_env_if_not_present(MEILI_SCHEDULE_SNAPSHOT, self.schedule_snapshot.to_string()); + export_to_env_if_not_present( + MEILI_SNAPSHOT_INTERVAL_SEC, + self.snapshot_interval_sec.to_string(), + ); + export_to_env_if_not_present(MEILI_DUMPS_DIR, self.dumps_dir); + export_to_env_if_not_present(MEILI_LOG_LEVEL, self.log_level); + #[cfg(feature = "metrics")] + { + export_to_env_if_not_present( + MEILI_ENABLE_METRICS_ROUTE, + self.enable_metrics_route.to_string(), + ); + } + self.indexer_options.export_to_env(); + self.scheduler_options.export_to_env(); + } + pub fn get_ssl_config(&self) -> anyhow::Result> { if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) { let config = rustls::ServerConfig::builder().with_safe_defaults(); diff --git a/meilisearch-lib/src/lib.rs b/meilisearch-lib/src/lib.rs index 70fd2ba51..7fe0984dc 100644 --- a/meilisearch-lib/src/lib.rs +++ b/meilisearch-lib/src/lib.rs @@ -11,6 +11,8 @@ mod snapshot; pub mod tasks; mod update_file_store; +use std::env::VarError; +use std::ffi::OsStr; use std::path::Path; pub use index_controller::MeiliSearch; @@ -35,3 +37,14 @@ pub fn is_empty_db(db_path: impl AsRef) -> bool { true } } + +/// Checks if the key is defined in the environment variables. +/// If not, inserts it with the given value. +pub fn export_to_env_if_not_present(key: &str, value: T) +where + T: AsRef, +{ + if let Err(VarError::NotPresent) = std::env::var(key) { + std::env::set_var(key, value); + } +} diff --git a/meilisearch-lib/src/options.rs b/meilisearch-lib/src/options.rs index ea810b9b7..0b9254848 100644 --- a/meilisearch-lib/src/options.rs +++ b/meilisearch-lib/src/options.rs @@ -1,22 +1,28 @@ +use crate::export_to_env_if_not_present; + use core::fmt; use std::{convert::TryFrom, num::ParseIntError, ops::Deref, str::FromStr}; use byte_unit::{Byte, ByteError}; use clap::Parser; use milli::update::IndexerConfig; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use sysinfo::{RefreshKind, System, SystemExt}; -#[derive(Debug, Clone, Parser, Serialize)] +const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY"; +const MEILI_MAX_INDEXING_THREADS: &str = "MEILI_MAX_INDEXING_THREADS"; +const DISABLE_AUTO_BATCHING: &str = "DISABLE_AUTO_BATCHING"; + +#[derive(Debug, Clone, Parser, Serialize, Deserialize)] pub struct IndexerOpts { /// The amount of documents to skip before printing /// a log regarding the indexing advancement. - #[serde(skip)] + #[serde(skip_serializing)] #[clap(long, default_value = "100000", hide = true)] // 100k pub log_every_n: usize, /// Grenad max number of chunks in bytes. - #[serde(skip)] + #[serde(skip_serializing)] #[clap(long, hide = true)] pub max_nb_chunks: Option, @@ -27,7 +33,7 @@ pub struct IndexerOpts { /// In case the engine is unable to retrieve the available memory the engine will /// try to use the memory it needs but without real limit, this can lead to /// Out-Of-Memory issues and it is recommended to specify the amount of memory to use. - #[clap(long, env = "MEILI_MAX_INDEXING_MEMORY", default_value_t)] + #[clap(long, env = MEILI_MAX_INDEXING_MEMORY, default_value_t)] pub max_indexing_memory: MaxMemory, /// The maximum number of threads the indexer will use. @@ -35,18 +41,33 @@ pub struct IndexerOpts { /// it will use the maximum number of available cores. /// /// It defaults to half of the available threads. - #[clap(long, env = "MEILI_MAX_INDEXING_THREADS", default_value_t)] + #[clap(long, env = MEILI_MAX_INDEXING_THREADS, default_value_t)] pub max_indexing_threads: MaxThreads, } -#[derive(Debug, Clone, Parser, Default, Serialize)] +#[derive(Debug, Clone, Parser, Default, Serialize, Deserialize)] pub struct SchedulerConfig { /// The engine will disable task auto-batching, /// and will sequencialy compute each task one by one. - #[clap(long, env = "DISABLE_AUTO_BATCHING")] + #[clap(long, env = DISABLE_AUTO_BATCHING)] pub disable_auto_batching: bool, } +impl IndexerOpts { + pub fn export_to_env(self) { + if let Some(max_indexing_memory) = self.max_indexing_memory.0 { + export_to_env_if_not_present( + MEILI_MAX_INDEXING_MEMORY, + max_indexing_memory.to_string(), + ); + } + export_to_env_if_not_present( + MEILI_MAX_INDEXING_THREADS, + self.max_indexing_threads.0.to_string(), + ); + } +} + impl TryFrom<&IndexerOpts> for IndexerConfig { type Error = anyhow::Error; @@ -77,8 +98,17 @@ impl Default for IndexerOpts { } } +impl SchedulerConfig { + pub fn export_to_env(self) { + export_to_env_if_not_present( + DISABLE_AUTO_BATCHING, + self.disable_auto_batching.to_string(), + ); + } +} + /// A type used to detect the max memory available and use 2/3 of it. -#[derive(Debug, Clone, Copy, Serialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct MaxMemory(Option); impl FromStr for MaxMemory { @@ -134,7 +164,7 @@ fn total_memory_bytes() -> Option { } } -#[derive(Debug, Clone, Copy, Serialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct MaxThreads(usize); impl FromStr for MaxThreads { From 6520d3c4741f93bb38b0ecebdf8325eb335812b2 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Tue, 6 Sep 2022 14:50:49 +0200 Subject: [PATCH 02/15] Refactor build method and flag --- meilisearch-http/src/option.rs | 40 ++++++++++++++++------------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 143dfb231..9c1fc5732 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use std::sync::Arc; use byte_unit::Byte; -use clap::{Arg, Command, Parser}; +use clap::Parser; use meilisearch_lib::{ export_to_env_if_not_present, options::{IndexerOpts, SchedulerConfig}, @@ -193,6 +193,12 @@ pub struct Opt { #[serde(flatten)] #[clap(flatten)] pub scheduler_options: SchedulerConfig, + + /// The path to a configuration file that should be used to setup the engine. + /// Format must be TOML. + #[serde(skip_serializing)] + #[clap(long)] + config_file_path: Option, } impl Opt { @@ -203,29 +209,21 @@ impl Opt { } pub fn build() -> Self { - let args = Command::new("config") - .arg( - Arg::new("config_file_path") - .long("config-file-path") - .takes_value(true) - .default_value("./config.toml") - .help("Path to a config file, must be TOML format"), - ) - .get_matches(); - let config_file_path = args - .value_of("config_file_path") - .expect("default value present"); - if let Some(Ok(opts_from_file)) = match std::fs::read_to_string(config_file_path) { - Ok(config_str) => Some(toml::from_str::(&config_str)), - Err(err) => { - log::debug!("can't read {} : {}", config_file_path, err); - None + let mut opts = Opt::parse(); + if let Some(config_file_path) = opts.config_file_path.as_ref() { + eprintln!("loading config file : {:?}", config_file_path); + match std::fs::read(config_file_path) { + Ok(config) => { + let opt_from_config = + toml::from_slice::(&config).expect("can't read file"); + opt_from_config.export_to_env(); + opts = Opt::parse(); + } + Err(err) => eprintln!("can't read {:?} : {}", config_file_path, err), } - } { - opts_from_file.export_to_env(); } - Opt::parse() + opts } fn export_to_env(self) { From ef3fa925367d6c116ab8e6bdf3293255cf166f03 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Wed, 7 Sep 2022 11:51:23 +0200 Subject: [PATCH 03/15] Refactor default values for clap and serde --- meilisearch-http/src/option.rs | 98 ++++++++++++++++++++++++++++------ meilisearch-lib/src/options.rs | 13 ++++- 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 9c1fc5732..28ea39162 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -45,15 +45,28 @@ const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL"; #[cfg(feature = "metrics")] const MEILI_ENABLE_METRICS_ROUTE: &str = "MEILI_ENABLE_METRICS_ROUTE"; +const DEFAULT_DB_PATH: &str = "./data.ms"; +const DEFAULT_HTTP_ADDR: &str = "127.0.0.1:7700"; +const DEFAULT_ENV: &str = "development"; +const DEFAULT_MAX_INDEX_SIZE: &str = "100 GiB"; +const DEFAULT_MAX_TASK_DB_SIZE: &str = "100 GiB"; +const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB"; +const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/"; +const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400; +const DEFAULT_DUMPS_DIR: &str = "dumps/"; +const DEFAULT_LOG_LEVEL: &str = "info"; + #[derive(Debug, Clone, Parser, Serialize, Deserialize)] #[clap(version)] pub struct Opt { /// The destination where the database must be created. - #[clap(long, env = MEILI_DB_PATH, default_value = "./data.ms")] + #[clap(long, env = MEILI_DB_PATH, default_value_os_t = default_db_path())] + #[serde(default = "default_db_path")] pub db_path: PathBuf, /// The address on which the http server will listen. - #[clap(long, env = MEILI_HTTP_ADDR, default_value = "127.0.0.1:7700")] + #[clap(long, env = MEILI_HTTP_ADDR, default_value_t = default_http_addr())] + #[serde(default = "default_http_addr")] pub http_addr: String, /// The master key allowing you to do everything on the server. @@ -65,25 +78,29 @@ pub struct Opt { /// If the server is running in development mode more logs will be displayed, /// and the master key can be avoided which implies that there is no security on the updates routes. /// This is useful to debug when integrating the engine with another service. - #[clap(long, env = MEILI_ENV, default_value = "development", possible_values = &POSSIBLE_ENV)] + #[clap(long, env = MEILI_ENV, default_value_t = default_env(), possible_values = &POSSIBLE_ENV)] + #[serde(default = "default_env")] pub env: String, /// Do not send analytics to Meili. #[cfg(all(not(debug_assertions), feature = "analytics"))] - #[serde(skip_serializing)] // we can't send true + #[serde(skip_serializing, default)] // we can't send true #[clap(long, env = MEILI_NO_ANALYTICS)] pub no_analytics: bool, /// The maximum size, in bytes, of the main lmdb database directory - #[clap(long, env = MEILI_MAX_INDEX_SIZE, default_value = "100 GiB")] + #[clap(long, env = MEILI_MAX_INDEX_SIZE, default_value_t = default_max_index_size())] + #[serde(default = "default_max_index_size")] pub max_index_size: Byte, /// The maximum size, in bytes, of the update lmdb database directory - #[clap(long, env = MEILI_MAX_TASK_DB_SIZE, default_value = "100 GiB")] + #[clap(long, env = MEILI_MAX_TASK_DB_SIZE, default_value_t = default_max_task_db_size())] + #[serde(default = "default_max_task_db_size")] pub max_task_db_size: Byte, /// The maximum size, in bytes, of accepted JSON payloads - #[clap(long, env = MEILI_HTTP_PAYLOAD_SIZE_LIMIT, default_value = "100 MB")] + #[clap(long, env = MEILI_HTTP_PAYLOAD_SIZE_LIMIT, default_value_t = default_http_payload_size_limit())] + #[serde(default = "default_http_payload_size_limit")] pub http_payload_size_limit: Byte, /// Read server certificates from CERTFILE. @@ -113,17 +130,17 @@ pub struct Opt { pub ssl_ocsp_path: Option, /// Send a fatal alert if the client does not complete client authentication. - #[serde(skip_serializing)] + #[serde(skip_serializing, default)] #[clap(long, env = MEILI_SSL_REQUIRE_AUTH)] pub ssl_require_auth: bool, /// SSL support session resumption - #[serde(skip_serializing)] + #[serde(skip_serializing, default)] #[clap(long, env = MEILI_SSL_RESUMPTION)] pub ssl_resumption: bool, /// SSL support tickets. - #[serde(skip_serializing)] + #[serde(skip_serializing, default)] #[clap(long, env = MEILI_SSL_TICKETS)] pub ssl_tickets: bool, @@ -139,6 +156,7 @@ pub struct Opt { env = "MEILI_IGNORE_MISSING_SNAPSHOT", requires = "import-snapshot" )] + #[serde(default)] pub ignore_missing_snapshot: bool, /// The engine will skip snapshot importation and not return an error in such case. @@ -147,18 +165,23 @@ pub struct Opt { env = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS", requires = "import-snapshot" )] + #[serde(default)] pub ignore_snapshot_if_db_exists: bool, /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap. - #[clap(long, env = MEILI_SNAPSHOT_DIR, default_value = "snapshots/")] + #[clap(long, env = MEILI_SNAPSHOT_DIR, default_value_os_t = default_snapshot_dir())] + #[serde(default = "default_snapshot_dir")] pub snapshot_dir: PathBuf, /// Activate snapshot scheduling. #[clap(long, env = MEILI_SCHEDULE_SNAPSHOT)] + #[serde(default)] pub schedule_snapshot: bool, /// Defines time interval, in seconds, between each snapshot creation. - #[clap(long, env = MEILI_SNAPSHOT_INTERVAL_SEC, default_value = "86400")] // 24h + #[clap(long, env = MEILI_SNAPSHOT_INTERVAL_SEC, default_value_t = default_snapshot_interval_sec())] + #[serde(default = "default_snapshot_interval_sec")] + // 24h pub snapshot_interval_sec: u64, /// Import a dump from the specified path, must be a `.dump` file. @@ -167,23 +190,28 @@ pub struct Opt { /// If the dump doesn't exists, load or create the database specified by `db-path` instead. #[clap(long, env = "MEILI_IGNORE_MISSING_DUMP", requires = "import-dump")] + #[serde(default)] pub ignore_missing_dump: bool, /// Ignore the dump if a database already exists, and load that database instead. #[clap(long, env = "MEILI_IGNORE_DUMP_IF_DB_EXISTS", requires = "import-dump")] + #[serde(default)] pub ignore_dump_if_db_exists: bool, /// Folder where dumps are created when the dump route is called. - #[clap(long, env = MEILI_DUMPS_DIR, default_value = "dumps/")] + #[clap(long, env = MEILI_DUMPS_DIR, default_value_os_t = default_dumps_dir())] + #[serde(default = "default_dumps_dir")] pub dumps_dir: PathBuf, /// Set the log level - #[clap(long, env = MEILI_LOG_LEVEL, default_value = "info")] + #[clap(long, env = MEILI_LOG_LEVEL, default_value_t = default_log_level())] + #[serde(default = "default_log_level")] pub log_level: String, /// Enables Prometheus metrics and /metrics route. #[cfg(feature = "metrics")] #[clap(long, env = MEILI_ENABLE_METRICS_ROUTE)] + #[serde(default)] pub enable_metrics_route: bool, #[serde(flatten)] @@ -235,7 +263,7 @@ impl Opt { export_to_env_if_not_present(MEILI_ENV, self.env); #[cfg(all(not(debug_assertions), feature = "analytics"))] { - export_to_env_if_not_present(MEILI_NO_ANALYTICS, self.no_analytics); + export_to_env_if_not_present(MEILI_NO_ANALYTICS, self.no_analytics.to_string()); } export_to_env_if_not_present(MEILI_MAX_INDEX_SIZE, self.max_index_size.to_string()); export_to_env_if_not_present(MEILI_MAX_TASK_DB_SIZE, self.max_task_db_size.to_string()); @@ -375,6 +403,46 @@ fn load_ocsp(filename: &Option) -> anyhow::Result> { Ok(ret) } +fn default_db_path() -> PathBuf { + PathBuf::from(DEFAULT_DB_PATH) +} + +fn default_http_addr() -> String { + DEFAULT_HTTP_ADDR.to_string() +} + +fn default_env() -> String { + DEFAULT_ENV.to_string() +} + +fn default_max_index_size() -> Byte { + Byte::from_str(DEFAULT_MAX_INDEX_SIZE).unwrap() +} + +fn default_max_task_db_size() -> Byte { + Byte::from_str(DEFAULT_MAX_TASK_DB_SIZE).unwrap() +} + +fn default_http_payload_size_limit() -> Byte { + Byte::from_str(DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT).unwrap() +} + +fn default_snapshot_dir() -> PathBuf { + PathBuf::from(DEFAULT_SNAPSHOT_DIR) +} + +fn default_snapshot_interval_sec() -> u64 { + DEFAULT_SNAPSHOT_INTERVAL_SEC +} + +fn default_dumps_dir() -> PathBuf { + PathBuf::from(DEFAULT_DUMPS_DIR) +} + +fn default_log_level() -> String { + DEFAULT_LOG_LEVEL.to_string() +} + #[cfg(test)] mod test { use super::*; diff --git a/meilisearch-lib/src/options.rs b/meilisearch-lib/src/options.rs index 0b9254848..4e208187d 100644 --- a/meilisearch-lib/src/options.rs +++ b/meilisearch-lib/src/options.rs @@ -13,12 +13,14 @@ const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY"; const MEILI_MAX_INDEXING_THREADS: &str = "MEILI_MAX_INDEXING_THREADS"; const DISABLE_AUTO_BATCHING: &str = "DISABLE_AUTO_BATCHING"; +const DEFAULT_LOG_EVERY_N: usize = 100000; + #[derive(Debug, Clone, Parser, Serialize, Deserialize)] pub struct IndexerOpts { /// The amount of documents to skip before printing /// a log regarding the indexing advancement. - #[serde(skip_serializing)] - #[clap(long, default_value = "100000", hide = true)] // 100k + #[serde(skip_serializing, default = "default_log_every_n")] + #[clap(long, default_value_t = default_log_every_n(), hide = true)] // 100k pub log_every_n: usize, /// Grenad max number of chunks in bytes. @@ -34,6 +36,7 @@ pub struct IndexerOpts { /// try to use the memory it needs but without real limit, this can lead to /// Out-Of-Memory issues and it is recommended to specify the amount of memory to use. #[clap(long, env = MEILI_MAX_INDEXING_MEMORY, default_value_t)] + #[serde(default)] pub max_indexing_memory: MaxMemory, /// The maximum number of threads the indexer will use. @@ -42,6 +45,7 @@ pub struct IndexerOpts { /// /// It defaults to half of the available threads. #[clap(long, env = MEILI_MAX_INDEXING_THREADS, default_value_t)] + #[serde(default)] pub max_indexing_threads: MaxThreads, } @@ -50,6 +54,7 @@ pub struct SchedulerConfig { /// The engine will disable task auto-batching, /// and will sequencialy compute each task one by one. #[clap(long, env = DISABLE_AUTO_BATCHING)] + #[serde(default)] pub disable_auto_batching: bool, } @@ -194,3 +199,7 @@ impl Deref for MaxThreads { &self.0 } } + +fn default_log_every_n() -> usize { + DEFAULT_LOG_EVERY_N +} From 135499f398e34be3c970c7da6165d0533c0ed759 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Wed, 7 Sep 2022 17:47:15 +0200 Subject: [PATCH 04/15] Extract new env vars to const --- meilisearch-http/src/option.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 28ea39162..c16856b58 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -37,9 +37,15 @@ const MEILI_SSL_OCSP_PATH: &str = "MEILI_SSL_OCSP_PATH"; const MEILI_SSL_REQUIRE_AUTH: &str = "MEILI_SSL_REQUIRE_AUTH"; const MEILI_SSL_RESUMPTION: &str = "MEILI_SSL_RESUMPTION"; const MEILI_SSL_TICKETS: &str = "MEILI_SSL_TICKETS"; +const MEILI_IMPORT_SNAPSHOT: &str = "MEILI_IMPORT_SNAPSHOT"; +const MEILI_IGNORE_MISSING_SNAPSHOT: &str = "MEILI_IGNORE_MISSING_SNAPSHOT"; +const MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS: &str = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS"; const MEILI_SNAPSHOT_DIR: &str = "MEILI_SNAPSHOT_DIR"; const MEILI_SCHEDULE_SNAPSHOT: &str = "MEILI_SCHEDULE_SNAPSHOT"; const MEILI_SNAPSHOT_INTERVAL_SEC: &str = "MEILI_SNAPSHOT_INTERVAL_SEC"; +const MEILI_IMPORT_DUMP: &str = "MEILI_IMPORT_DUMP"; +const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP"; +const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS"; const MEILI_DUMPS_DIR: &str = "MEILI_DUMPS_DIR"; const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL"; #[cfg(feature = "metrics")] @@ -147,13 +153,13 @@ pub struct Opt { /// Defines the path of the snapshot file to import. /// This option will, by default, stop the process if a database already exist or if no snapshot exists at /// the given path. If this option is not specified no snapshot is imported. - #[clap(long, env = "MEILI_IMPORT_SNAPSHOT")] + #[clap(long, env = MEILI_IMPORT_SNAPSHOT)] pub import_snapshot: Option, /// The engine will ignore a missing snapshot and not return an error in such case. #[clap( long, - env = "MEILI_IGNORE_MISSING_SNAPSHOT", + env = MEILI_IGNORE_MISSING_SNAPSHOT, requires = "import-snapshot" )] #[serde(default)] @@ -162,7 +168,7 @@ pub struct Opt { /// The engine will skip snapshot importation and not return an error in such case. #[clap( long, - env = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS", + env = MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS, requires = "import-snapshot" )] #[serde(default)] @@ -185,16 +191,16 @@ pub struct Opt { pub snapshot_interval_sec: u64, /// Import a dump from the specified path, must be a `.dump` file. - #[clap(long, env = "MEILI_IMPORT_DUMP", conflicts_with = "import-snapshot")] + #[clap(long, env = MEILI_IMPORT_DUMP, conflicts_with = "import-snapshot")] pub import_dump: Option, /// If the dump doesn't exists, load or create the database specified by `db-path` instead. - #[clap(long, env = "MEILI_IGNORE_MISSING_DUMP", requires = "import-dump")] + #[clap(long, env = MEILI_IGNORE_MISSING_DUMP, requires = "import-dump")] #[serde(default)] pub ignore_missing_dump: bool, /// Ignore the dump if a database already exists, and load that database instead. - #[clap(long, env = "MEILI_IGNORE_DUMP_IF_DB_EXISTS", requires = "import-dump")] + #[clap(long, env = MEILI_IGNORE_DUMP_IF_DB_EXISTS, requires = "import-dump")] #[serde(default)] pub ignore_dump_if_db_exists: bool, From 5a4f1508d0a09e8a7e5ad123deb8c96bf9f9e413 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Wed, 7 Sep 2022 18:16:33 +0200 Subject: [PATCH 05/15] Add documentation --- meilisearch-http/src/option.rs | 8 ++++++++ meilisearch-lib/src/options.rs | 1 + 2 files changed, 9 insertions(+) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index c16856b58..3351e8b92 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -242,15 +242,20 @@ impl Opt { !self.no_analytics } + /// Build a new Opt from config file, env vars and cli args. pub fn build() -> Self { + // Parse the args to get the config_file_path. let mut opts = Opt::parse(); if let Some(config_file_path) = opts.config_file_path.as_ref() { eprintln!("loading config file : {:?}", config_file_path); match std::fs::read(config_file_path) { Ok(config) => { + // If the arg is present, and the file successfully read, we deserialize it with `toml`. let opt_from_config = toml::from_slice::(&config).expect("can't read file"); + // We inject the values from the toml in the corresponding env vars if needs be. Doing so, we respect the priority toml < env vars < cli args. opt_from_config.export_to_env(); + // Once injected we parse the cli args once again to take the new env vars into scope. opts = Opt::parse(); } Err(err) => eprintln!("can't read {:?} : {}", config_file_path, err), @@ -260,6 +265,7 @@ impl Opt { opts } + /// Exports the opts values to their corresponding env vars if they are not set. fn export_to_env(self) { export_to_env_if_not_present(MEILI_DB_PATH, self.db_path); export_to_env_if_not_present(MEILI_HTTP_ADDR, self.http_addr); @@ -409,6 +415,8 @@ fn load_ocsp(filename: &Option) -> anyhow::Result> { Ok(ret) } +/// Functions used to get default value for `Opt` fields, needs to be function because of serde's default attribute. + fn default_db_path() -> PathBuf { PathBuf::from(DEFAULT_DB_PATH) } diff --git a/meilisearch-lib/src/options.rs b/meilisearch-lib/src/options.rs index 4e208187d..5aa7edf37 100644 --- a/meilisearch-lib/src/options.rs +++ b/meilisearch-lib/src/options.rs @@ -59,6 +59,7 @@ pub struct SchedulerConfig { } impl IndexerOpts { + /// Exports the values to their corresponding env vars if they are not set. pub fn export_to_env(self) { if let Some(max_indexing_memory) = self.max_indexing_memory.0 { export_to_env_if_not_present( From 7f267ec4be1901102be5154222545500ff34fb2a Mon Sep 17 00:00:00 2001 From: mlemesle Date: Wed, 7 Sep 2022 20:22:49 +0200 Subject: [PATCH 06/15] Fix clippy --- meilisearch-http/src/option.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 3351e8b92..11a396904 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -232,7 +232,7 @@ pub struct Opt { /// Format must be TOML. #[serde(skip_serializing)] #[clap(long)] - config_file_path: Option, + pub config_file_path: Option, } impl Opt { From 579fa3f1add4730cad01c6c8e9b69e8f1c82d2d1 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Thu, 8 Sep 2022 11:05:52 +0200 Subject: [PATCH 07/15] Remove unnecessary println --- meilisearch-http/src/main.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index e74c1e056..147f526a2 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -30,8 +30,6 @@ fn setup(opt: &Opt) -> anyhow::Result<()> { async fn main() -> anyhow::Result<()> { let opt = Opt::build(); - println!("{:?}", opt); - setup(&opt)?; match opt.env.as_ref() { From a690ace36e31c27a36a02e472fa95a4aee540f3d Mon Sep 17 00:00:00 2001 From: mlemesle Date: Fri, 9 Sep 2022 09:37:23 +0200 Subject: [PATCH 08/15] Add example config.toml with default values --- config.toml | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 config.toml diff --git a/config.toml b/config.toml new file mode 100644 index 000000000..dcfaf835d --- /dev/null +++ b/config.toml @@ -0,0 +1,121 @@ +# This file shows the default configuration of Meilisearch. +# All variables are defined here https://docs.meilisearch.com/learn/configuration/instance_options.html#environment-variables + + +### DUMP + +# Folder where dumps are created when the dump route is called +# dumps-dir = "dumps/" + +# Ignore the dump if a database already exists, and load that database instead +# ignore-dump-if-db-exists = false + +# If the dump doesn't exists, load or create the database specified by `db-path` instead +# ignore-missing-dump = false + +# Import a dump from the specified path, must be a `.dump` file +# import-dump = "./path/to/my/file.dump" + + +### SNAPSHOT + +# The engine will ignore a missing snapshot and not return an error in such case +# ignore-missing-snapshot = false + +# The engine will skip snapshot importation and not return an error in such case +# ignore-snapshot-if-db-exists = false + +# Defines the path of the snapshot file to import. This option will, by default, stop the +# process if a database already exist or if no snapshot exists at the given path. If this +# option is not specified no snapshot is imported +# import-snapshot = false + +# Activate snapshot scheduling +# schedule-snapshot = false + +# Defines the directory path where meilisearch will create snapshot each snapshot_time_gap +# snapshot-dir = "snapshots/" + +# Defines time interval, in seconds, between each snapshot creation +# snapshot-interval-sec = 86400 + + +### INDEX + +# The maximum size, in bytes, of the main lmdb database directory +# max-index-size = "100 GiB" + +# The maximum amount of memory the indexer will use. It defaults to 2/3 of the available +# memory. It is recommended to use something like 80%-90% of the available memory, no +# more. +# +# In case the engine is unable to retrieve the available memory the engine will try to use +# the memory it needs but without real limit, this can lead to Out-Of-Memory issues and it +# is recommended to specify the amount of memory to use. +# +# /!\ The default value is system dependant /!\ +# max-indexing-memory = "2 GiB" + +# The maximum number of threads the indexer will use. If the number set is higher than the +# real number of cores available in the machine, it will use the maximum number of +# available cores. +# +# It defaults to half of the available threads. +# max-indexing-threads = 4 + + +### SSL + +# Enable client authentication, and accept certificates signed by those roots provided in CERTFILE +# ssl-auth-path = "./path/to/root" + +# Read server certificates from CERTFILE. This should contain PEM-format certificates in +# the right order (the first certificate should certify KEYFILE, the last should be a root +# CA) +# ssl-cert-path = "./path/to/CERTFILE" + +# Read private key from KEYFILE. This should be a RSA private key or PKCS8-encoded +# private key, in PEM format +# ssl-key-path = "./path/to/private-key" + +# Read DER-encoded OCSP response from OCSPFILE and staple to certificate. Optional +# ssl-ocsp-path = "./path/to/OCSPFILE" + +# Send a fatal alert if the client does not complete client authentication +# ssl-require-auth = false + +# SSL support session resumption +# ssl-resumption = false + +# SSL support tickets +# ssl-tickets = false + + +### MISC + +# This environment variable must be set to `production` if you are running in production. +# If the server is running in development mode more logs will be displayed, and the master +# key can be avoided which implies that there is no security on the updates routes. This +# is useful to debug when integrating the engine with another service +# env = "development" # possible values: [development, production] + +# The address on which the http server will listen +# http-addr = "127.0.0.1:7700" + +# The maximum size, in bytes, of accepted JSON payloads +# http-payload-size-limit = 100000000 + +# The destination where the database must be created +# db-path = "./data.ms" + +# The engine will disable task auto-batching, and will sequencialy compute each task one by one +# disable-auto-batching = false + +# Set the log level +# log-level = "info" + +# The master key allowing you to do everything on the server +# master-key = "YOUR MASTER KEY" + +# The maximum size, in bytes, of the update lmdb database directory +# max-task-db-size = "100 GiB" From 4dfae444780981550231404a19cd81eb117a0196 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Mon, 19 Sep 2022 18:16:28 +0200 Subject: [PATCH 09/15] Apply PR review comments --- config.toml | 156 ++++++++++++++++++--------------- meilisearch-http/src/main.rs | 13 ++- meilisearch-http/src/option.rs | 145 +++++++++++++++++++----------- meilisearch-lib/src/options.rs | 21 +++-- 4 files changed, 203 insertions(+), 132 deletions(-) diff --git a/config.toml b/config.toml index dcfaf835d..5d5ae4507 100644 --- a/config.toml +++ b/config.toml @@ -1,50 +1,55 @@ # This file shows the default configuration of Meilisearch. # All variables are defined here https://docs.meilisearch.com/learn/configuration/instance_options.html#environment-variables +# db_path = "./data.ms" +# The destination where the database must be created. + +# env = "development" # Possible values: [development, production] +# This environment variable must be set to `production` if you are running in production. +# More logs wiil be displayed if the server is running in development mode. Setting the master +# key is optional; hence no security on the updates routes. This +# is useful to debug when integrating the engine with another service. + +# http_addr = "127.0.0.1:7700" +# The address on which the HTTP server will listen. + +# master-key = "MASTER_KEY" +# Sets the instance's master key, automatically protecting all routes except GET /health. + +# no_analytics = false +# Do not send analytics to Meilisearch. + +# disable-auto-batching = false +# The engine will disable task auto-batching, and will sequencialy compute each task one by one. + ### DUMP -# Folder where dumps are created when the dump route is called # dumps-dir = "dumps/" +# Folder where dumps are created when the dump route is called. -# Ignore the dump if a database already exists, and load that database instead -# ignore-dump-if-db-exists = false - -# If the dump doesn't exists, load or create the database specified by `db-path` instead -# ignore-missing-dump = false - -# Import a dump from the specified path, must be a `.dump` file # import-dump = "./path/to/my/file.dump" +# Import a dump from the specified path, must be a `.dump` file. + +# ignore-missing-dump = false +# If the dump doesn't exist, load or create the database specified by `db-path` instead. + +# ignore-dump-if-db-exists = false +# Ignore the dump if a database already exists, and load that database instead. + +### -### SNAPSHOT - -# The engine will ignore a missing snapshot and not return an error in such case -# ignore-missing-snapshot = false - -# The engine will skip snapshot importation and not return an error in such case -# ignore-snapshot-if-db-exists = false - -# Defines the path of the snapshot file to import. This option will, by default, stop the -# process if a database already exist or if no snapshot exists at the given path. If this -# option is not specified no snapshot is imported -# import-snapshot = false - -# Activate snapshot scheduling -# schedule-snapshot = false - -# Defines the directory path where meilisearch will create snapshot each snapshot_time_gap -# snapshot-dir = "snapshots/" - -# Defines time interval, in seconds, between each snapshot creation -# snapshot-interval-sec = 86400 +# log-level = "INFO" # Possible values: [ERROR, WARN, INFO, DEBUG, TRACE] +# Set the log level. ### INDEX -# The maximum size, in bytes, of the main lmdb database directory # max-index-size = "100 GiB" +# The maximum size, in bytes, of the main LMDB database directory. +# max-indexing-memory = "2 GiB" # The maximum amount of memory the indexer will use. It defaults to 2/3 of the available # memory. It is recommended to use something like 80%-90% of the available memory, no # more. @@ -54,68 +59,73 @@ # is recommended to specify the amount of memory to use. # # /!\ The default value is system dependant /!\ -# max-indexing-memory = "2 GiB" +# max-indexing-threads = 4 # The maximum number of threads the indexer will use. If the number set is higher than the # real number of cores available in the machine, it will use the maximum number of # available cores. # # It defaults to half of the available threads. -# max-indexing-threads = 4 + +### + + +# max-task-db-size = "100 GiB" +# The maximum size, in bytes, of the update LMDB database directory. + +# http-payload-size-limit = 100000000 +# The maximum size, in bytes, of accepted JSON payloads. + + +### SNAPSHOT + +# schedule-snapshot = false +# Activate snapshot scheduling. + +# snapshot-dir = "snapshots/" +# Defines the directory path where Meilisearch will create a snapshot each snapshot-interval-sec. + +# snapshot-interval-sec = 86400 +# Defines time interval, in seconds, between each snapshot creation. + +# import-snapshot = false +# Defines the path of the snapshot file to import. This option will, by default, stop the +# process if a database already exist, or if no snapshot exists at the given path. If this +# option is not specified, no snapshot is imported. + +# ignore-missing-snapshot = false +# The engine will ignore a missing snapshot and not return an error in such a case. + +# ignore-snapshot-if-db-exists = false +# The engine will skip snapshot importation and not return an error in such a case. + +### ### SSL -# Enable client authentication, and accept certificates signed by those roots provided in CERTFILE # ssl-auth-path = "./path/to/root" +# Enable client authentication, and accept certificates signed by those roots provided in CERTFILE. +# ssl-cert-path = "./path/to/CERTFILE" # Read server certificates from CERTFILE. This should contain PEM-format certificates in # the right order (the first certificate should certify KEYFILE, the last should be a root -# CA) -# ssl-cert-path = "./path/to/CERTFILE" +# CA). -# Read private key from KEYFILE. This should be a RSA private key or PKCS8-encoded -# private key, in PEM format # ssl-key-path = "./path/to/private-key" - -# Read DER-encoded OCSP response from OCSPFILE and staple to certificate. Optional +# Read the private key from KEYFILE. This should be an RSA private key or PKCS8-encoded +# private key, in PEM format. + # ssl-ocsp-path = "./path/to/OCSPFILE" +# Read DER-encoded OCSP response from OCSPFILE and staple to certificate. Optional. -# Send a fatal alert if the client does not complete client authentication # ssl-require-auth = false - -# SSL support session resumption +# Send a fatal alert if the client does not complete client authentication. + # ssl-resumption = false - -# SSL support tickets +# SSL support session resumption. + # ssl-tickets = false +# SSL support tickets. - -### MISC - -# This environment variable must be set to `production` if you are running in production. -# If the server is running in development mode more logs will be displayed, and the master -# key can be avoided which implies that there is no security on the updates routes. This -# is useful to debug when integrating the engine with another service -# env = "development" # possible values: [development, production] - -# The address on which the http server will listen -# http-addr = "127.0.0.1:7700" - -# The maximum size, in bytes, of accepted JSON payloads -# http-payload-size-limit = 100000000 - -# The destination where the database must be created -# db-path = "./data.ms" - -# The engine will disable task auto-batching, and will sequencialy compute each task one by one -# disable-auto-batching = false - -# Set the log level -# log-level = "info" - -# The master key allowing you to do everything on the server -# master-key = "YOUR MASTER KEY" - -# The maximum size, in bytes, of the update lmdb database directory -# max-task-db-size = "100 GiB" +### diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 147f526a2..01cf39a2f 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -1,4 +1,5 @@ use std::env; +use std::path::PathBuf; use std::sync::Arc; use actix_web::http::KeepAlive; @@ -28,7 +29,7 @@ fn setup(opt: &Opt) -> anyhow::Result<()> { #[actix_web::main] async fn main() -> anyhow::Result<()> { - let opt = Opt::build(); + let (opt, config_read_from) = Opt::try_build()?; setup(&opt)?; @@ -57,7 +58,7 @@ async fn main() -> anyhow::Result<()> { #[cfg(any(debug_assertions, not(feature = "analytics")))] let (analytics, user) = analytics::MockAnalytics::new(&opt); - print_launch_resume(&opt, &user); + print_launch_resume(&opt, &user, config_read_from); run_http(meilisearch, auth_controller, opt, analytics).await?; @@ -96,7 +97,7 @@ async fn run_http( Ok(()) } -pub fn print_launch_resume(opt: &Opt, user: &str) { +pub fn print_launch_resume(opt: &Opt, user: &str, config_read_from: Option) { let commit_sha = option_env!("VERGEN_GIT_SHA").unwrap_or("unknown"); let commit_date = option_env!("VERGEN_GIT_COMMIT_TIMESTAMP").unwrap_or("unknown"); let protocol = if opt.ssl_cert_path.is_some() && opt.ssl_key_path.is_some() { @@ -117,6 +118,12 @@ pub fn print_launch_resume(opt: &Opt, user: &str) { eprintln!("{}", ascii_name); + eprintln!( + "Config file path:\t{}", + config_read_from + .map(|config_file_path| config_file_path.display().to_string()) + .unwrap_or_else(|| "none".to_string()) + ); eprintln!("Database path:\t\t{:?}", opt.db_path); eprintln!("Server listening on:\t\"{}://{}\"", protocol, opt.http_addr); eprintln!("Environment:\t\t{:?}", opt.env); diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 11a396904..1f676813a 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -64,6 +64,7 @@ const DEFAULT_LOG_LEVEL: &str = "info"; #[derive(Debug, Clone, Parser, Serialize, Deserialize)] #[clap(version)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] pub struct Opt { /// The destination where the database must be created. #[clap(long, env = MEILI_DB_PATH, default_value_os_t = default_db_path())] @@ -75,15 +76,15 @@ pub struct Opt { #[serde(default = "default_http_addr")] pub http_addr: String, - /// The master key allowing you to do everything on the server. + /// Sets the instance's master key, automatically protecting all routes except GET /health #[serde(skip_serializing)] #[clap(long, env = MEILI_MASTER_KEY)] pub master_key: Option, /// This environment variable must be set to `production` if you are running in production. - /// If the server is running in development mode more logs will be displayed, - /// and the master key can be avoided which implies that there is no security on the updates routes. - /// This is useful to debug when integrating the engine with another service. + /// More logs wiil be displayed if the server is running in development mode. Setting the master + /// key is optional; hence no security on the updates routes. This + /// is useful to debug when integrating the engine with another service #[clap(long, env = MEILI_ENV, default_value_t = default_env(), possible_values = &POSSIBLE_ENV)] #[serde(default = "default_env")] pub env: String, @@ -94,12 +95,12 @@ pub struct Opt { #[clap(long, env = MEILI_NO_ANALYTICS)] pub no_analytics: bool, - /// The maximum size, in bytes, of the main lmdb database directory + /// The maximum size, in bytes, of the main LMDB database directory #[clap(long, env = MEILI_MAX_INDEX_SIZE, default_value_t = default_max_index_size())] #[serde(default = "default_max_index_size")] pub max_index_size: Byte, - /// The maximum size, in bytes, of the update lmdb database directory + /// The maximum size, in bytes, of the update LMDB database directory #[clap(long, env = MEILI_MAX_TASK_DB_SIZE, default_value_t = default_max_task_db_size())] #[serde(default = "default_max_task_db_size")] pub max_task_db_size: Byte, @@ -117,7 +118,7 @@ pub struct Opt { #[clap(long, env = MEILI_SSL_CERT_PATH, parse(from_os_str))] pub ssl_cert_path: Option, - /// Read private key from KEYFILE. This should be a RSA + /// Read the private key from KEYFILE. This should be an RSA /// private key or PKCS8-encoded private key, in PEM format. #[serde(skip_serializing)] #[clap(long, env = MEILI_SSL_KEY_PATH, parse(from_os_str))] @@ -151,12 +152,12 @@ pub struct Opt { pub ssl_tickets: bool, /// Defines the path of the snapshot file to import. - /// This option will, by default, stop the process if a database already exist or if no snapshot exists at - /// the given path. If this option is not specified no snapshot is imported. + /// This option will, by default, stop the process if a database already exists or if no snapshot exists at + /// the given path. If this option is not specified, no snapshot is imported. #[clap(long, env = MEILI_IMPORT_SNAPSHOT)] pub import_snapshot: Option, - /// The engine will ignore a missing snapshot and not return an error in such case. + /// The engine will ignore a missing snapshot and not return an error in such a case. #[clap( long, env = MEILI_IGNORE_MISSING_SNAPSHOT, @@ -174,7 +175,7 @@ pub struct Opt { #[serde(default)] pub ignore_snapshot_if_db_exists: bool, - /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap. + /// Defines the directory path where Meilisearch will create a snapshot each snapshot-interval-sec. #[clap(long, env = MEILI_SNAPSHOT_DIR, default_value_os_t = default_snapshot_dir())] #[serde(default = "default_snapshot_dir")] pub snapshot_dir: PathBuf, @@ -194,7 +195,7 @@ pub struct Opt { #[clap(long, env = MEILI_IMPORT_DUMP, conflicts_with = "import-snapshot")] pub import_dump: Option, - /// If the dump doesn't exists, load or create the database specified by `db-path` instead. + /// If the dump doesn't exist, load or create the database specified by `db-path` instead. #[clap(long, env = MEILI_IGNORE_MISSING_DUMP, requires = "import-dump")] #[serde(default)] pub ignore_missing_dump: bool, @@ -209,7 +210,7 @@ pub struct Opt { #[serde(default = "default_dumps_dir")] pub dumps_dir: PathBuf, - /// Set the log level + /// Set the log level. # Possible values: [ERROR, WARN, INFO, DEBUG, TRACE] #[clap(long, env = MEILI_LOG_LEVEL, default_value_t = default_log_level())] #[serde(default = "default_log_level")] pub log_level: String, @@ -243,78 +244,124 @@ impl Opt { } /// Build a new Opt from config file, env vars and cli args. - pub fn build() -> Self { + pub fn try_build() -> anyhow::Result<(Self, Option)> { // Parse the args to get the config_file_path. let mut opts = Opt::parse(); - if let Some(config_file_path) = opts.config_file_path.as_ref() { - eprintln!("loading config file : {:?}", config_file_path); - match std::fs::read(config_file_path) { + let mut config_read_from = None; + if let Some(config_file_path) = opts + .config_file_path + .clone() + .or_else(|| Some(PathBuf::from("./config.toml"))) + { + match std::fs::read(&config_file_path) { Ok(config) => { - // If the arg is present, and the file successfully read, we deserialize it with `toml`. - let opt_from_config = - toml::from_slice::(&config).expect("can't read file"); - // We inject the values from the toml in the corresponding env vars if needs be. Doing so, we respect the priority toml < env vars < cli args. - opt_from_config.export_to_env(); - // Once injected we parse the cli args once again to take the new env vars into scope. - opts = Opt::parse(); + // If the file is successfully read, we deserialize it with `toml`. + match toml::from_slice::(&config) { + Ok(opt_from_config) => { + // We inject the values from the toml in the corresponding env vars if needs be. Doing so, we respect the priority toml < env vars < cli args. + opt_from_config.export_to_env(); + // Once injected we parse the cli args once again to take the new env vars into scope. + opts = Opt::parse(); + config_read_from = Some(config_file_path); + } + // If we have an error deserializing the file defined by the user. + Err(err) if opts.config_file_path.is_some() => anyhow::bail!(err), + _ => (), + } } - Err(err) => eprintln!("can't read {:?} : {}", config_file_path, err), + // If we have an error while reading the file defined by the user. + Err(err) if opts.config_file_path.is_some() => anyhow::bail!(err), + _ => (), } } - opts + Ok((opts, config_read_from)) } /// Exports the opts values to their corresponding env vars if they are not set. fn export_to_env(self) { - export_to_env_if_not_present(MEILI_DB_PATH, self.db_path); - export_to_env_if_not_present(MEILI_HTTP_ADDR, self.http_addr); - if let Some(master_key) = self.master_key { + let Opt { + db_path, + http_addr, + master_key, + env, + max_index_size, + max_task_db_size, + http_payload_size_limit, + ssl_cert_path, + ssl_key_path, + ssl_auth_path, + ssl_ocsp_path, + ssl_require_auth, + ssl_resumption, + ssl_tickets, + snapshot_dir, + schedule_snapshot, + snapshot_interval_sec, + dumps_dir, + log_level, + indexer_options, + scheduler_options, + import_snapshot: _, + ignore_missing_snapshot: _, + ignore_snapshot_if_db_exists: _, + import_dump: _, + ignore_missing_dump: _, + ignore_dump_if_db_exists: _, + config_file_path: _, + #[cfg(all(not(debug_assertions), feature = "analytics"))] + no_analytics, + #[cfg(feature = "metrics")] + enable_metrics_route, + } = self; + export_to_env_if_not_present(MEILI_DB_PATH, db_path); + export_to_env_if_not_present(MEILI_HTTP_ADDR, http_addr); + if let Some(master_key) = master_key { export_to_env_if_not_present(MEILI_MASTER_KEY, master_key); } - export_to_env_if_not_present(MEILI_ENV, self.env); + export_to_env_if_not_present(MEILI_ENV, env); #[cfg(all(not(debug_assertions), feature = "analytics"))] { - export_to_env_if_not_present(MEILI_NO_ANALYTICS, self.no_analytics.to_string()); + export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string()); } - export_to_env_if_not_present(MEILI_MAX_INDEX_SIZE, self.max_index_size.to_string()); - export_to_env_if_not_present(MEILI_MAX_TASK_DB_SIZE, self.max_task_db_size.to_string()); + export_to_env_if_not_present(MEILI_MAX_INDEX_SIZE, max_index_size.to_string()); + export_to_env_if_not_present(MEILI_MAX_TASK_DB_SIZE, max_task_db_size.to_string()); export_to_env_if_not_present( MEILI_HTTP_PAYLOAD_SIZE_LIMIT, - self.http_payload_size_limit.to_string(), + http_payload_size_limit.to_string(), ); - if let Some(ssl_cert_path) = self.ssl_cert_path { + if let Some(ssl_cert_path) = ssl_cert_path { export_to_env_if_not_present(MEILI_SSL_CERT_PATH, ssl_cert_path); } - if let Some(ssl_key_path) = self.ssl_key_path { + if let Some(ssl_key_path) = ssl_key_path { export_to_env_if_not_present(MEILI_SSL_KEY_PATH, ssl_key_path); } - if let Some(ssl_auth_path) = self.ssl_auth_path { + if let Some(ssl_auth_path) = ssl_auth_path { export_to_env_if_not_present(MEILI_SSL_AUTH_PATH, ssl_auth_path); } - if let Some(ssl_ocsp_path) = self.ssl_ocsp_path { + if let Some(ssl_ocsp_path) = ssl_ocsp_path { export_to_env_if_not_present(MEILI_SSL_OCSP_PATH, ssl_ocsp_path); } - export_to_env_if_not_present(MEILI_SSL_REQUIRE_AUTH, self.ssl_require_auth.to_string()); - export_to_env_if_not_present(MEILI_SSL_RESUMPTION, self.ssl_resumption.to_string()); - export_to_env_if_not_present(MEILI_SSL_TICKETS, self.ssl_tickets.to_string()); - export_to_env_if_not_present(MEILI_SNAPSHOT_DIR, self.snapshot_dir); - export_to_env_if_not_present(MEILI_SCHEDULE_SNAPSHOT, self.schedule_snapshot.to_string()); + export_to_env_if_not_present(MEILI_SSL_REQUIRE_AUTH, ssl_require_auth.to_string()); + export_to_env_if_not_present(MEILI_SSL_RESUMPTION, ssl_resumption.to_string()); + export_to_env_if_not_present(MEILI_SSL_TICKETS, ssl_tickets.to_string()); + export_to_env_if_not_present(MEILI_SNAPSHOT_DIR, snapshot_dir); + export_to_env_if_not_present(MEILI_SCHEDULE_SNAPSHOT, schedule_snapshot.to_string()); export_to_env_if_not_present( MEILI_SNAPSHOT_INTERVAL_SEC, - self.snapshot_interval_sec.to_string(), + snapshot_interval_sec.to_string(), ); - export_to_env_if_not_present(MEILI_DUMPS_DIR, self.dumps_dir); - export_to_env_if_not_present(MEILI_LOG_LEVEL, self.log_level); + export_to_env_if_not_present(MEILI_DUMPS_DIR, dumps_dir); + export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level); #[cfg(feature = "metrics")] { export_to_env_if_not_present( MEILI_ENABLE_METRICS_ROUTE, - self.enable_metrics_route.to_string(), + enable_metrics_route.to_string(), ); } - self.indexer_options.export_to_env(); - self.scheduler_options.export_to_env(); + indexer_options.export_to_env(); + scheduler_options.export_to_env(); } pub fn get_ssl_config(&self) -> anyhow::Result> { diff --git a/meilisearch-lib/src/options.rs b/meilisearch-lib/src/options.rs index 5aa7edf37..d75e02b39 100644 --- a/meilisearch-lib/src/options.rs +++ b/meilisearch-lib/src/options.rs @@ -12,10 +12,10 @@ use sysinfo::{RefreshKind, System, SystemExt}; const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY"; const MEILI_MAX_INDEXING_THREADS: &str = "MEILI_MAX_INDEXING_THREADS"; const DISABLE_AUTO_BATCHING: &str = "DISABLE_AUTO_BATCHING"; - const DEFAULT_LOG_EVERY_N: usize = 100000; #[derive(Debug, Clone, Parser, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] pub struct IndexerOpts { /// The amount of documents to skip before printing /// a log regarding the indexing advancement. @@ -50,6 +50,7 @@ pub struct IndexerOpts { } #[derive(Debug, Clone, Parser, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] pub struct SchedulerConfig { /// The engine will disable task auto-batching, /// and will sequencialy compute each task one by one. @@ -61,7 +62,13 @@ pub struct SchedulerConfig { impl IndexerOpts { /// Exports the values to their corresponding env vars if they are not set. pub fn export_to_env(self) { - if let Some(max_indexing_memory) = self.max_indexing_memory.0 { + let IndexerOpts { + max_indexing_memory, + max_indexing_threads, + log_every_n: _, + max_nb_chunks: _, + } = self; + if let Some(max_indexing_memory) = max_indexing_memory.0 { export_to_env_if_not_present( MEILI_MAX_INDEXING_MEMORY, max_indexing_memory.to_string(), @@ -69,7 +76,7 @@ impl IndexerOpts { } export_to_env_if_not_present( MEILI_MAX_INDEXING_THREADS, - self.max_indexing_threads.0.to_string(), + max_indexing_threads.0.to_string(), ); } } @@ -106,10 +113,10 @@ impl Default for IndexerOpts { impl SchedulerConfig { pub fn export_to_env(self) { - export_to_env_if_not_present( - DISABLE_AUTO_BATCHING, - self.disable_auto_batching.to_string(), - ); + let SchedulerConfig { + disable_auto_batching, + } = self; + export_to_env_if_not_present(DISABLE_AUTO_BATCHING, disable_auto_batching.to_string()); } } From d406fe901b365a8b035bfb27fa2a5331d1d0bd71 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Wed, 21 Sep 2022 10:55:16 +0200 Subject: [PATCH 10/15] Pass config.toml keys to snake_case --- config.toml | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/config.toml b/config.toml index 5d5ae4507..6d7f1af14 100644 --- a/config.toml +++ b/config.toml @@ -13,43 +13,43 @@ # http_addr = "127.0.0.1:7700" # The address on which the HTTP server will listen. -# master-key = "MASTER_KEY" +# master_key = "MASTER_KEY" # Sets the instance's master key, automatically protecting all routes except GET /health. # no_analytics = false # Do not send analytics to Meilisearch. -# disable-auto-batching = false +# disable_auto_batching = false # The engine will disable task auto-batching, and will sequencialy compute each task one by one. ### DUMP -# dumps-dir = "dumps/" +# dumps_dir = "dumps/" # Folder where dumps are created when the dump route is called. -# import-dump = "./path/to/my/file.dump" +# import_dump = "./path/to/my/file.dump" # Import a dump from the specified path, must be a `.dump` file. -# ignore-missing-dump = false -# If the dump doesn't exist, load or create the database specified by `db-path` instead. +# ignore_missing_dump = false +# If the dump doesn't exist, load or create the database specified by `db_path` instead. -# ignore-dump-if-db-exists = false +# ignore_dump_if_db_exists = false # Ignore the dump if a database already exists, and load that database instead. ### -# log-level = "INFO" # Possible values: [ERROR, WARN, INFO, DEBUG, TRACE] +# log_level = "INFO" # Possible values: [ERROR, WARN, INFO, DEBUG, TRACE] # Set the log level. ### INDEX -# max-index-size = "100 GiB" +# max_index_size = "100 GiB" # The maximum size, in bytes, of the main LMDB database directory. -# max-indexing-memory = "2 GiB" +# max_indexing_memory = "2 GiB" # The maximum amount of memory the indexer will use. It defaults to 2/3 of the available # memory. It is recommended to use something like 80%-90% of the available memory, no # more. @@ -60,7 +60,7 @@ # # /!\ The default value is system dependant /!\ -# max-indexing-threads = 4 +# max_indexing_threads = 4 # The maximum number of threads the indexer will use. If the number set is higher than the # real number of cores available in the machine, it will use the maximum number of # available cores. @@ -70,33 +70,33 @@ ### -# max-task-db-size = "100 GiB" +# max_task_db_size = "100 GiB" # The maximum size, in bytes, of the update LMDB database directory. -# http-payload-size-limit = 100000000 +# http_payload_size_limit = 100000000 # The maximum size, in bytes, of accepted JSON payloads. ### SNAPSHOT -# schedule-snapshot = false +# schedule_snapshot = false # Activate snapshot scheduling. -# snapshot-dir = "snapshots/" -# Defines the directory path where Meilisearch will create a snapshot each snapshot-interval-sec. +# snapshot_dir = "snapshots/" +# Defines the directory path where Meilisearch will create a snapshot each snapshot_interval_sec. -# snapshot-interval-sec = 86400 +# snapshot_interval_sec = 86400 # Defines time interval, in seconds, between each snapshot creation. -# import-snapshot = false +# import_snapshot = false # Defines the path of the snapshot file to import. This option will, by default, stop the # process if a database already exist, or if no snapshot exists at the given path. If this # option is not specified, no snapshot is imported. -# ignore-missing-snapshot = false +# ignore_missing_snapshot = false # The engine will ignore a missing snapshot and not return an error in such a case. -# ignore-snapshot-if-db-exists = false +# ignore_snapshot_if_db_exists = false # The engine will skip snapshot importation and not return an error in such a case. ### @@ -104,28 +104,28 @@ ### SSL -# ssl-auth-path = "./path/to/root" +# ssl_auth_path = "./path/to/root" # Enable client authentication, and accept certificates signed by those roots provided in CERTFILE. -# ssl-cert-path = "./path/to/CERTFILE" +# ssl_cert_path = "./path/to/CERTFILE" # Read server certificates from CERTFILE. This should contain PEM-format certificates in # the right order (the first certificate should certify KEYFILE, the last should be a root # CA). -# ssl-key-path = "./path/to/private-key" +# ssl_key_path = "./path/to/private-key" # Read the private key from KEYFILE. This should be an RSA private key or PKCS8-encoded # private key, in PEM format. -# ssl-ocsp-path = "./path/to/OCSPFILE" +# ssl_ocsp_path = "./path/to/OCSPFILE" # Read DER-encoded OCSP response from OCSPFILE and staple to certificate. Optional. -# ssl-require-auth = false +# ssl_require_auth = false # Send a fatal alert if the client does not complete client authentication. -# ssl-resumption = false +# ssl_resumption = false # SSL support session resumption. -# ssl-tickets = false +# ssl_tickets = false # SSL support tickets. ### From 740926e7474a603f77c770488265d331032ad3b7 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Tue, 20 Sep 2022 22:39:35 +0800 Subject: [PATCH 11/15] Fix typos Found via `codespell -L crate,nam,hart,succeded`. --- .github/scripts/is-latest-release.sh | 2 +- .github/workflows/milestone-workflow.yml | 12 ++++++------ .github/workflows/publish-docker-images.yml | 2 +- CONTRIBUTING.md | 2 +- meilisearch-http/src/analytics/segment_analytics.rs | 10 +++++----- meilisearch-http/src/option.rs | 2 +- meilisearch-lib/src/dump/compat/v2.rs | 2 +- meilisearch-lib/src/dump/loaders/v4.rs | 4 ++-- meilisearch-lib/src/index/search.rs | 2 +- meilisearch-lib/src/index_resolver/index_store.rs | 2 +- meilisearch-lib/src/tasks/task_store/store.rs | 2 +- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/scripts/is-latest-release.sh b/.github/scripts/is-latest-release.sh index 81534a2f7..54f0a9d3a 100644 --- a/.github/scripts/is-latest-release.sh +++ b/.github/scripts/is-latest-release.sh @@ -85,7 +85,7 @@ get_latest() { latest="" current_tag="" for release_info in $releases; do - if [ $i -eq 0 ]; then # Cheking tag_name + if [ $i -eq 0 ]; then # Checking tag_name if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release current_tag=$release_info else diff --git a/.github/workflows/milestone-workflow.yml b/.github/workflows/milestone-workflow.yml index 4d0425f14..4cb87684d 100644 --- a/.github/workflows/milestone-workflow.yml +++ b/.github/workflows/milestone-workflow.yml @@ -62,12 +62,12 @@ jobs: - uses: actions/checkout@v3 - name: Download the issue template run: curl -s https://raw.githubusercontent.com/meilisearch/core-team/main/issue-templates/roadmap-issue.md > $ISSUE_TEMPLATE - - name: Replace all empty occurences in the templates + - name: Replace all empty occurrences in the templates run: | - # Replace all <> occurences + # Replace all <> occurrences sed -i "s/<>/$MILESTONE_VERSION/g" $ISSUE_TEMPLATE - # Replace all <> occurences + # Replace all <> occurrences milestone_id=$(echo $MILESTONE_URL | cut -d '/' -f 7) sed -i "s/<>/$milestone_id/g" $ISSUE_TEMPLATE @@ -95,12 +95,12 @@ jobs: - uses: actions/checkout@v3 - name: Download the issue template run: curl -s https://raw.githubusercontent.com/meilisearch/core-team/main/issue-templates/changelog-issue.md > $ISSUE_TEMPLATE - - name: Replace all empty occurences in the templates + - name: Replace all empty occurrences in the templates run: | - # Replace all <> occurences + # Replace all <> occurrences sed -i "s/<>/$MILESTONE_VERSION/g" $ISSUE_TEMPLATE - # Replace all <> occurences + # Replace all <> occurrences milestone_id=$(echo $MILESTONE_URL | cut -d '/' -f 7) sed -i "s/<>/$milestone_id/g" $ISSUE_TEMPLATE - name: Create the issue diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index 449aec020..f2e119a6d 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -53,7 +53,7 @@ jobs: uses: docker/metadata-action@v4 with: images: getmeili/meilisearch - # The lastest and `vX.Y` tags are only pushed for the official Meilisearch releases + # The latest and `vX.Y` tags are only pushed for the official Meilisearch releases # See https://github.com/docker/metadata-action#latest-tag flavor: latest=false tags: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c40c7dac..bf433eb09 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,7 +102,7 @@ The full Meilisearch release process is described in [this guide](https://github ### Release assets For each release, the following assets are created: -- Binaries for differents platforms (Linux, MacOS, Windows and ARM architectures) are attached to the GitHub release +- Binaries for different platforms (Linux, MacOS, Windows and ARM architectures) are attached to the GitHub release - Binaries are pushed to HomeBrew and APT (not published for RC) - Docker tags are created/updated: - `vX.Y.Z` diff --git a/meilisearch-http/src/analytics/segment_analytics.rs b/meilisearch-http/src/analytics/segment_analytics.rs index f0dfd0fab..7b76cdd80 100644 --- a/meilisearch-http/src/analytics/segment_analytics.rs +++ b/meilisearch-http/src/analytics/segment_analytics.rs @@ -349,16 +349,16 @@ pub struct SearchAggregator { // sort sort_with_geo_point: bool, - // everytime a request has a filter, this field must be incremented by the number of terms it contains + // every time a request has a filter, this field must be incremented by the number of terms it contains sort_sum_of_criteria_terms: usize, - // everytime a request has a filter, this field must be incremented by one + // every time a request has a filter, this field must be incremented by one sort_total_number_of_criteria: usize, // filter filter_with_geo_radius: bool, - // everytime a request has a filter, this field must be incremented by the number of terms it contains + // every time a request has a filter, this field must be incremented by the number of terms it contains filter_sum_of_criteria_terms: usize, - // everytime a request has a filter, this field must be incremented by one + // every time a request has a filter, this field must be incremented by one filter_total_number_of_criteria: usize, used_syntax: HashMap, @@ -366,7 +366,7 @@ pub struct SearchAggregator { // The maximum number of terms in a q request max_terms_number: usize, - // everytime a search is done, we increment the counter linked to the used settings + // every time a search is done, we increment the counter linked to the used settings matching_strategy: HashMap, // pagination diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index bdfa283a6..31942aeec 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -169,7 +169,7 @@ pub struct Opt { } impl Opt { - /// Wether analytics should be enabled or not. + /// Whether analytics should be enabled or not. #[cfg(all(not(debug_assertions), feature = "analytics"))] pub fn analytics(&self) -> bool { !self.no_analytics diff --git a/meilisearch-lib/src/dump/compat/v2.rs b/meilisearch-lib/src/dump/compat/v2.rs index 364d894c4..ba3b8e3a6 100644 --- a/meilisearch-lib/src/dump/compat/v2.rs +++ b/meilisearch-lib/src/dump/compat/v2.rs @@ -145,7 +145,7 @@ pub fn error_code_from_str(s: &str) -> anyhow::Result { "unsupported_media_type" => Code::UnsupportedMediaType, "dump_already_in_progress" => Code::DumpAlreadyInProgress, "dump_process_failed" => Code::DumpProcessFailed, - _ => bail!("unknow error code."), + _ => bail!("unknown error code."), }; Ok(code) diff --git a/meilisearch-lib/src/dump/loaders/v4.rs b/meilisearch-lib/src/dump/loaders/v4.rs index 0744df7ea..44ec23517 100644 --- a/meilisearch-lib/src/dump/loaders/v4.rs +++ b/meilisearch-lib/src/dump/loaders/v4.rs @@ -57,10 +57,10 @@ fn patch_updates(src: impl AsRef, dst: impl AsRef) -> anyhow::Result let updates_path = src.as_ref().join("updates/data.jsonl"); let output_updates_path = dst.as_ref().join("updates/data.jsonl"); create_dir_all(output_updates_path.parent().unwrap())?; - let udpates_file = File::open(updates_path)?; + let updates_file = File::open(updates_path)?; let mut output_update_file = File::create(output_updates_path)?; - serde_json::Deserializer::from_reader(udpates_file) + serde_json::Deserializer::from_reader(updates_file) .into_iter::() .try_for_each(|task| -> anyhow::Result<()> { let task: Task = task?.into(); diff --git a/meilisearch-lib/src/index/search.rs b/meilisearch-lib/src/index/search.rs index 57171d529..1a9aa1d0d 100644 --- a/meilisearch-lib/src/index/search.rs +++ b/meilisearch-lib/src/index/search.rs @@ -27,7 +27,7 @@ pub const DEFAULT_CROP_MARKER: fn() -> String = || "…".to_string(); pub const DEFAULT_HIGHLIGHT_PRE_TAG: fn() -> String = || "".to_string(); pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "".to_string(); -/// The maximimum number of results that the engine +/// The maximum number of results that the engine /// will be able to return in one search call. pub const DEFAULT_PAGINATION_MAX_TOTAL_HITS: usize = 1000; diff --git a/meilisearch-lib/src/index_resolver/index_store.rs b/meilisearch-lib/src/index_resolver/index_store.rs index e4f58f130..ea3c7125a 100644 --- a/meilisearch-lib/src/index_resolver/index_store.rs +++ b/meilisearch-lib/src/index_resolver/index_store.rs @@ -51,7 +51,7 @@ impl MapIndexStore { #[async_trait::async_trait] impl IndexStore for MapIndexStore { async fn create(&self, uuid: Uuid) -> Result { - // We need to keep the lock until we are sure the db file has been opened correclty, to + // We need to keep the lock until we are sure the db file has been opened correctly, to // ensure that another db is not created at the same time. let mut lock = self.index_store.write().await; diff --git a/meilisearch-lib/src/tasks/task_store/store.rs b/meilisearch-lib/src/tasks/task_store/store.rs index 24d0d3a65..32b20aeb8 100644 --- a/meilisearch-lib/src/tasks/task_store/store.rs +++ b/meilisearch-lib/src/tasks/task_store/store.rs @@ -63,7 +63,7 @@ impl Store { /// Returns the id for the next task. /// /// The required `mut txn` acts as a reservation system. It guarantees that as long as you commit - /// the task to the store in the same transaction, no one else will hav this task id. + /// the task to the store in the same transaction, no one else will have this task id. pub fn next_task_id(&self, txn: &mut RwTxn) -> Result { let id = self .tasks From 56d72d449337ee6c39b51d753d0d3127d937d299 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Wed, 21 Sep 2022 16:31:16 +0200 Subject: [PATCH 12/15] Uncomment static default values and fix typo --- config.toml | 46 ++++++++++++++++------------------ meilisearch-http/src/option.rs | 6 ++--- meilisearch-lib/src/options.rs | 4 +-- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/config.toml b/config.toml index 6d7f1af14..8da71c70a 100644 --- a/config.toml +++ b/config.toml @@ -1,16 +1,16 @@ # This file shows the default configuration of Meilisearch. # All variables are defined here https://docs.meilisearch.com/learn/configuration/instance_options.html#environment-variables -# db_path = "./data.ms" +db_path = "./data.ms" # The destination where the database must be created. -# env = "development" # Possible values: [development, production] +env = "development" # Possible values: [development, production] # This environment variable must be set to `production` if you are running in production. # More logs wiil be displayed if the server is running in development mode. Setting the master # key is optional; hence no security on the updates routes. This # is useful to debug when integrating the engine with another service. -# http_addr = "127.0.0.1:7700" +http_addr = "127.0.0.1:7700" # The address on which the HTTP server will listen. # master_key = "MASTER_KEY" @@ -19,40 +19,38 @@ # no_analytics = false # Do not send analytics to Meilisearch. -# disable_auto_batching = false +disable_auto_batching = false # The engine will disable task auto-batching, and will sequencialy compute each task one by one. ### DUMP -# dumps_dir = "dumps/" +dumps_dir = "dumps/" # Folder where dumps are created when the dump route is called. # import_dump = "./path/to/my/file.dump" # Import a dump from the specified path, must be a `.dump` file. -# ignore_missing_dump = false +ignore_missing_dump = false # If the dump doesn't exist, load or create the database specified by `db_path` instead. -# ignore_dump_if_db_exists = false +ignore_dump_if_db_exists = false # Ignore the dump if a database already exists, and load that database instead. ### -# log_level = "INFO" # Possible values: [ERROR, WARN, INFO, DEBUG, TRACE] +log_level = "INFO" # Possible values: [ERROR, WARN, INFO, DEBUG, TRACE] # Set the log level. ### INDEX -# max_index_size = "100 GiB" +max_index_size = "100 GiB" # The maximum size, in bytes, of the main LMDB database directory. # max_indexing_memory = "2 GiB" -# The maximum amount of memory the indexer will use. It defaults to 2/3 of the available -# memory. It is recommended to use something like 80%-90% of the available memory, no -# more. +# The maximum amount of memory the indexer will use. # # In case the engine is unable to retrieve the available memory the engine will try to use # the memory it needs but without real limit, this can lead to Out-Of-Memory issues and it @@ -70,33 +68,33 @@ ### -# max_task_db_size = "100 GiB" +max_task_db_size = "100 GiB" # The maximum size, in bytes, of the update LMDB database directory. -# http_payload_size_limit = 100000000 +http_payload_size_limit = "100 MB" # The maximum size, in bytes, of accepted JSON payloads. ### SNAPSHOT -# schedule_snapshot = false +schedule_snapshot = false # Activate snapshot scheduling. -# snapshot_dir = "snapshots/" +snapshot_dir = "snapshots/" # Defines the directory path where Meilisearch will create a snapshot each snapshot_interval_sec. -# snapshot_interval_sec = 86400 +snapshot_interval_sec = 86400 # Defines time interval, in seconds, between each snapshot creation. -# import_snapshot = false +# import_snapshot = "./path/to/my/snapshot" # Defines the path of the snapshot file to import. This option will, by default, stop the -# process if a database already exist, or if no snapshot exists at the given path. If this +# process if a database already exists, or if no snapshot exists at the given path. If this # option is not specified, no snapshot is imported. -# ignore_missing_snapshot = false +ignore_missing_snapshot = false # The engine will ignore a missing snapshot and not return an error in such a case. -# ignore_snapshot_if_db_exists = false +ignore_snapshot_if_db_exists = false # The engine will skip snapshot importation and not return an error in such a case. ### @@ -119,13 +117,13 @@ # ssl_ocsp_path = "./path/to/OCSPFILE" # Read DER-encoded OCSP response from OCSPFILE and staple to certificate. Optional. -# ssl_require_auth = false +ssl_require_auth = false # Send a fatal alert if the client does not complete client authentication. -# ssl_resumption = false +ssl_resumption = false # SSL support session resumption. -# ssl_tickets = false +ssl_tickets = false # SSL support tickets. ### diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 1f676813a..ff8c0d120 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -60,7 +60,7 @@ const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB"; const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/"; const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400; const DEFAULT_DUMPS_DIR: &str = "dumps/"; -const DEFAULT_LOG_LEVEL: &str = "info"; +const DEFAULT_LOG_LEVEL: &str = "INFO"; #[derive(Debug, Clone, Parser, Serialize, Deserialize)] #[clap(version)] @@ -126,8 +126,8 @@ pub struct Opt { /// Enable client authentication, and accept certificates /// signed by those roots provided in CERTFILE. - #[clap(long, env = MEILI_SSL_AUTH_PATH, parse(from_os_str))] #[serde(skip_serializing)] + #[clap(long, env = MEILI_SSL_AUTH_PATH, parse(from_os_str))] pub ssl_auth_path: Option, /// Read DER-encoded OCSP response from OCSPFILE and staple to certificate. @@ -152,7 +152,7 @@ pub struct Opt { pub ssl_tickets: bool, /// Defines the path of the snapshot file to import. - /// This option will, by default, stop the process if a database already exists or if no snapshot exists at + /// This option will, by default, stop the process if a database already exists, or if no snapshot exists at /// the given path. If this option is not specified, no snapshot is imported. #[clap(long, env = MEILI_IMPORT_SNAPSHOT)] pub import_snapshot: Option, diff --git a/meilisearch-lib/src/options.rs b/meilisearch-lib/src/options.rs index d75e02b39..bd406fbdd 100644 --- a/meilisearch-lib/src/options.rs +++ b/meilisearch-lib/src/options.rs @@ -28,9 +28,7 @@ pub struct IndexerOpts { #[clap(long, hide = true)] pub max_nb_chunks: Option, - /// The maximum amount of memory the indexer will use. It defaults to 2/3 - /// of the available memory. It is recommended to use something like 80%-90% - /// of the available memory, no more. + /// The maximum amount of memory the indexer will use. /// /// In case the engine is unable to retrieve the available memory the engine will /// try to use the memory it needs but without real limit, this can lead to From 248d727e6137c42e73b4f0a1d15ae0d7e90404b8 Mon Sep 17 00:00:00 2001 From: mlemesle Date: Thu, 22 Sep 2022 09:44:28 +0200 Subject: [PATCH 13/15] Add quotes around file name and change error message --- meilisearch-http/src/main.rs | 2 +- meilisearch-http/src/option.rs | 23 ++++++++++------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 01cf39a2f..b6f92ae28 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -119,7 +119,7 @@ pub fn print_launch_resume(opt: &Opt, user: &str, config_read_from: Option { // If the file is successfully read, we deserialize it with `toml`. - match toml::from_slice::(&config) { - Ok(opt_from_config) => { - // We inject the values from the toml in the corresponding env vars if needs be. Doing so, we respect the priority toml < env vars < cli args. - opt_from_config.export_to_env(); - // Once injected we parse the cli args once again to take the new env vars into scope. - opts = Opt::parse(); - config_read_from = Some(config_file_path); - } - // If we have an error deserializing the file defined by the user. - Err(err) if opts.config_file_path.is_some() => anyhow::bail!(err), - _ => (), - } + let opt_from_config = toml::from_slice::(&config)?; + // We inject the values from the toml in the corresponding env vars if needs be. Doing so, we respect the priority toml < env vars < cli args. + opt_from_config.export_to_env(); + // Once injected we parse the cli args once again to take the new env vars into scope. + opts = Opt::parse(); + config_read_from = Some(config_file_path); } // If we have an error while reading the file defined by the user. - Err(err) if opts.config_file_path.is_some() => anyhow::bail!(err), + Err(_) if opts.config_file_path.is_some() => anyhow::bail!( + "unable to open or read the {:?} configuration file.", + opts.config_file_path.unwrap().display().to_string() + ), _ => (), } } From 05f93541d833c6b90bb95a773da338e3838371b3 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Thu, 29 Sep 2022 01:42:10 +0200 Subject: [PATCH 14/15] Skip dashboard test if mini-dashboard feature is disabled Fixes the following error: cargo test --no-default-features ... error: couldn't read target/debug/build/meilisearch-http-ec029d8c902cf2cb/out/generated.rs: No such file or directory (os error 2) --> meilisearch-http/tests/dashboard/mod.rs:8:9 | 8 | include!(concat!(env!("OUT_DIR"), "/generated.rs")); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `include` (in Nightly builds, run with -Z macro-backtrace for more info) error: could not compile `meilisearch-http` due to previous error --- meilisearch-http/tests/dashboard/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/meilisearch-http/tests/dashboard/mod.rs b/meilisearch-http/tests/dashboard/mod.rs index d097cfd4b..2699cd16f 100644 --- a/meilisearch-http/tests/dashboard/mod.rs +++ b/meilisearch-http/tests/dashboard/mod.rs @@ -1,5 +1,6 @@ use crate::common::Server; +#[cfg(feature = "mini-dashboard")] #[actix_rt::test] async fn dashboard_assets_load() { let server = Server::new().await; From 20589a41b5fa0590b6a9511e01ef5fed3e46000d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaras=C5=82a=C5=AD=20Viktor=C4=8Dyk?= Date: Sat, 1 Oct 2022 21:59:20 +0200 Subject: [PATCH 15/15] Rename receivedDocumentIds into matchedDocuments Changes DocumentDeletion task details response. --- meilisearch-http/src/task.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/task.rs b/meilisearch-http/src/task.rs index fe23720aa..786d318f8 100644 --- a/meilisearch-http/src/task.rs +++ b/meilisearch-http/src/task.rs @@ -147,7 +147,7 @@ enum TaskDetails { IndexInfo { primary_key: Option }, #[serde(rename_all = "camelCase")] DocumentDeletion { - received_document_ids: usize, + matched_documents: usize, deleted_documents: Option, }, #[serde(rename_all = "camelCase")] @@ -255,7 +255,7 @@ impl From for TaskView { } => ( TaskType::DocumentDeletion, Some(TaskDetails::DocumentDeletion { - received_document_ids: ids.len(), + matched_documents: ids.len(), deleted_documents: None, }), ),