mirror of
https://github.com/meilisearch/meilisearch.git
synced 2024-11-22 18:17:39 +08:00
Merge #2068
2068: chore(http): migrate from structopt to clap3 r=Kerollmops a=MarinPostma migrate from structopt to clap3 This fix the long lasting issue with flags require a value, such as `--no-analytics` or `--schedule-snapshot`. All flag arguments now take NO argument, i.e: `meilisearch --schedule-snapshot true` becomes `meilisearch --schedule-snapshot` as per https://docs.rs/clap/latest/clap/struct.Arg.html#method.env, the env variable is defines as: > A false literal is n, no, f, false, off or 0. An absent environment variable will also be considered as false. Anything else will considered as true. `@gmourier` `@curquiza` `@meilisearch/docs-team` Co-authored-by: mpostma <postma.marin@protonmail.com>
This commit is contained in:
commit
f9f075bca2
101
Cargo.lock
generated
101
Cargo.lock
generated
@ -278,15 +278,6 @@ dependencies = [
|
|||||||
"memchr",
|
"memchr",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "ansi_term"
|
|
||||||
version = "0.12.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
|
|
||||||
dependencies = [
|
|
||||||
"winapi",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "anyhow"
|
name = "anyhow"
|
||||||
version = "1.0.51"
|
version = "1.0.51"
|
||||||
@ -639,17 +630,32 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap"
|
name = "clap"
|
||||||
version = "2.34.0"
|
version = "3.0.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
|
checksum = "1957aa4a5fb388f0a0a73ce7556c5b42025b874e5cdc2c670775e346e97adec0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ansi_term",
|
|
||||||
"atty",
|
"atty",
|
||||||
"bitflags",
|
"bitflags",
|
||||||
|
"clap_derive",
|
||||||
|
"indexmap",
|
||||||
|
"lazy_static",
|
||||||
|
"os_str_bytes",
|
||||||
"strsim",
|
"strsim",
|
||||||
|
"termcolor",
|
||||||
"textwrap",
|
"textwrap",
|
||||||
"unicode-width",
|
]
|
||||||
"vec_map",
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap_derive"
|
||||||
|
version = "3.0.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "517358c28fcef6607bf6f76108e02afad7e82297d132a6b846dcc1fc3efcd153"
|
||||||
|
dependencies = [
|
||||||
|
"heck",
|
||||||
|
"proc-macro-error",
|
||||||
|
"proc-macro2 1.0.32",
|
||||||
|
"quote 1.0.10",
|
||||||
|
"syn 1.0.82",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -1258,12 +1264,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "heck"
|
name = "heck"
|
||||||
version = "0.3.3"
|
version = "0.4.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
|
checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
|
||||||
dependencies = [
|
|
||||||
"unicode-segmentation",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "heed"
|
name = "heed"
|
||||||
@ -1668,6 +1671,7 @@ dependencies = [
|
|||||||
"bytes",
|
"bytes",
|
||||||
"cargo_toml",
|
"cargo_toml",
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"clap",
|
||||||
"crossbeam-channel",
|
"crossbeam-channel",
|
||||||
"either",
|
"either",
|
||||||
"env_logger",
|
"env_logger",
|
||||||
@ -1706,7 +1710,6 @@ dependencies = [
|
|||||||
"sha2",
|
"sha2",
|
||||||
"siphasher",
|
"siphasher",
|
||||||
"slice-group-by",
|
"slice-group-by",
|
||||||
"structopt",
|
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
"tar",
|
"tar",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
@ -1734,6 +1737,7 @@ dependencies = [
|
|||||||
"byte-unit",
|
"byte-unit",
|
||||||
"bytes",
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"clap",
|
||||||
"crossbeam-channel",
|
"crossbeam-channel",
|
||||||
"csv",
|
"csv",
|
||||||
"derivative",
|
"derivative",
|
||||||
@ -1772,7 +1776,6 @@ dependencies = [
|
|||||||
"serde_json",
|
"serde_json",
|
||||||
"siphasher",
|
"siphasher",
|
||||||
"slice-group-by",
|
"slice-group-by",
|
||||||
"structopt",
|
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
"tar",
|
"tar",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
@ -2055,6 +2058,15 @@ dependencies = [
|
|||||||
"num-traits",
|
"num-traits",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "os_str_bytes"
|
||||||
|
version = "6.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "page_size"
|
name = "page_size"
|
||||||
version = "0.4.2"
|
version = "0.4.2"
|
||||||
@ -2890,33 +2902,9 @@ checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "strsim"
|
name = "strsim"
|
||||||
version = "0.8.0"
|
version = "0.10.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
|
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "structopt"
|
|
||||||
version = "0.3.25"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"
|
|
||||||
dependencies = [
|
|
||||||
"clap",
|
|
||||||
"lazy_static",
|
|
||||||
"structopt-derive",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "structopt-derive"
|
|
||||||
version = "0.4.18"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
|
|
||||||
dependencies = [
|
|
||||||
"heck",
|
|
||||||
"proc-macro-error",
|
|
||||||
"proc-macro2 1.0.32",
|
|
||||||
"quote 1.0.10",
|
|
||||||
"syn 1.0.82",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "syn"
|
name = "syn"
|
||||||
@ -3018,12 +3006,9 @@ checksum = "13a4ec180a2de59b57434704ccfad967f789b12737738798fa08798cd5824c16"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "textwrap"
|
name = "textwrap"
|
||||||
version = "0.11.0"
|
version = "0.14.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
|
checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
|
||||||
dependencies = [
|
|
||||||
"unicode-width",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "thiserror"
|
name = "thiserror"
|
||||||
@ -3274,12 +3259,6 @@ version = "1.8.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
|
checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "unicode-width"
|
|
||||||
version = "0.1.9"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-xid"
|
name = "unicode-xid"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@ -3338,12 +3317,6 @@ version = "0.2.15"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "vec_map"
|
|
||||||
version = "0.8.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "vergen"
|
name = "vergen"
|
||||||
version = "5.1.18"
|
version = "5.1.18"
|
||||||
|
@ -67,7 +67,7 @@ serde_json = { version = "1.0.67", features = ["preserve_order"] }
|
|||||||
sha2 = "0.9.6"
|
sha2 = "0.9.6"
|
||||||
siphasher = "0.3.7"
|
siphasher = "0.3.7"
|
||||||
slice-group-by = "0.2.6"
|
slice-group-by = "0.2.6"
|
||||||
structopt = "0.3.25"
|
clap = { version = "3.0", features = ["derive", "env"] }
|
||||||
sysinfo = "0.20.2"
|
sysinfo = "0.20.2"
|
||||||
tar = "0.4.37"
|
tar = "0.4.37"
|
||||||
tempfile = "3.2.0"
|
tempfile = "3.2.0"
|
||||||
|
@ -2,12 +2,12 @@ use std::env;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use actix_web::HttpServer;
|
use actix_web::HttpServer;
|
||||||
|
use clap::Parser;
|
||||||
use meilisearch_auth::AuthController;
|
use meilisearch_auth::AuthController;
|
||||||
use meilisearch_http::analytics;
|
use meilisearch_http::analytics;
|
||||||
use meilisearch_http::analytics::Analytics;
|
use meilisearch_http::analytics::Analytics;
|
||||||
use meilisearch_http::{create_app, setup_meilisearch, Opt};
|
use meilisearch_http::{create_app, setup_meilisearch, Opt};
|
||||||
use meilisearch_lib::MeiliSearch;
|
use meilisearch_lib::MeiliSearch;
|
||||||
use structopt::StructOpt;
|
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
#[global_allocator]
|
#[global_allocator]
|
||||||
@ -29,7 +29,7 @@ fn setup(opt: &Opt) -> anyhow::Result<()> {
|
|||||||
|
|
||||||
#[actix_web::main]
|
#[actix_web::main]
|
||||||
async fn main() -> anyhow::Result<()> {
|
async fn main() -> anyhow::Result<()> {
|
||||||
let opt = Opt::from_args();
|
let opt = Opt::parse();
|
||||||
|
|
||||||
setup(&opt)?;
|
setup(&opt)?;
|
||||||
|
|
||||||
@ -50,7 +50,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
let auth_controller = AuthController::new(&opt.db_path, &opt.master_key)?;
|
let auth_controller = AuthController::new(&opt.db_path, &opt.master_key)?;
|
||||||
|
|
||||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
||||||
let (analytics, user) = if opt.analytics() {
|
let (analytics, user) = if !opt.no_analytics {
|
||||||
analytics::SegmentAnalytics::new(&opt, &meilisearch).await
|
analytics::SegmentAnalytics::new(&opt, &meilisearch).await
|
||||||
} else {
|
} else {
|
||||||
analytics::MockAnalytics::new(&opt)
|
analytics::MockAnalytics::new(&opt)
|
||||||
@ -125,7 +125,7 @@ pub fn print_launch_resume(opt: &Opt, user: &str) {
|
|||||||
|
|
||||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
||||||
{
|
{
|
||||||
if opt.analytics() {
|
if !opt.no_analytics {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"
|
"
|
||||||
Thank you for using MeiliSearch!
|
Thank you for using MeiliSearch!
|
||||||
|
@ -4,141 +4,131 @@ use std::path::PathBuf;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use byte_unit::Byte;
|
use byte_unit::Byte;
|
||||||
|
use clap::Parser;
|
||||||
use meilisearch_lib::options::IndexerOpts;
|
use meilisearch_lib::options::IndexerOpts;
|
||||||
use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
|
use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
|
||||||
use rustls::{
|
use rustls::{
|
||||||
AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
|
AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
|
||||||
RootCertStore,
|
RootCertStore,
|
||||||
};
|
};
|
||||||
use structopt::StructOpt;
|
|
||||||
|
|
||||||
const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
|
const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
|
||||||
|
|
||||||
#[derive(Debug, Clone, StructOpt)]
|
#[derive(Debug, Clone, Parser)]
|
||||||
pub struct Opt {
|
pub struct Opt {
|
||||||
/// The destination where the database must be created.
|
/// The destination where the database must be created.
|
||||||
#[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
|
#[clap(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
|
||||||
pub db_path: PathBuf,
|
pub db_path: PathBuf,
|
||||||
|
|
||||||
/// The address on which the http server will listen.
|
/// The address on which the http server will listen.
|
||||||
#[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")]
|
#[clap(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")]
|
||||||
pub http_addr: String,
|
pub http_addr: String,
|
||||||
|
|
||||||
/// The master key allowing you to do everything on the server.
|
/// The master key allowing you to do everything on the server.
|
||||||
#[structopt(long, env = "MEILI_MASTER_KEY")]
|
#[clap(long, env = "MEILI_MASTER_KEY")]
|
||||||
pub master_key: Option<String>,
|
pub master_key: Option<String>,
|
||||||
|
|
||||||
/// This environment variable must be set to `production` if you are running in production.
|
/// This environment variable must be set to `production` if you are running in production.
|
||||||
/// If the server is running in development mode more logs will be displayed,
|
/// If the server is running in development mode more logs will be displayed,
|
||||||
/// and the master key can be avoided which implies that there is no security on the updates routes.
|
/// and the master key can be avoided which implies that there is no security on the updates routes.
|
||||||
/// This is useful to debug when integrating the engine with another service.
|
/// This is useful to debug when integrating the engine with another service.
|
||||||
#[structopt(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)]
|
#[clap(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)]
|
||||||
pub env: String,
|
pub env: String,
|
||||||
|
|
||||||
/// Do not send analytics to Meili.
|
/// Do not send analytics to Meili.
|
||||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
||||||
#[structopt(long, env = "MEILI_NO_ANALYTICS")]
|
#[clap(long, env = "MEILI_NO_ANALYTICS")]
|
||||||
pub no_analytics: Option<Option<bool>>,
|
pub no_analytics: bool,
|
||||||
|
|
||||||
/// The maximum size, in bytes, of the main lmdb database directory
|
/// The maximum size, in bytes, of the main lmdb database directory
|
||||||
#[structopt(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")]
|
#[clap(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")]
|
||||||
pub max_index_size: Byte,
|
pub max_index_size: Byte,
|
||||||
|
|
||||||
/// The maximum size, in bytes, of the update lmdb database directory
|
/// The maximum size, in bytes, of the update lmdb database directory
|
||||||
#[structopt(long, env = "MEILI_MAX_TASK_DB_SIZE", default_value = "100 GiB")]
|
#[clap(long, env = "MEILI_MAX_TASK_DB_SIZE", default_value = "100 GiB")]
|
||||||
pub max_task_db_size: Byte,
|
pub max_task_db_size: Byte,
|
||||||
|
|
||||||
/// The maximum size, in bytes, of accepted JSON payloads
|
/// The maximum size, in bytes, of accepted JSON payloads
|
||||||
#[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")]
|
#[clap(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")]
|
||||||
pub http_payload_size_limit: Byte,
|
pub http_payload_size_limit: Byte,
|
||||||
|
|
||||||
/// Read server certificates from CERTFILE.
|
/// Read server certificates from CERTFILE.
|
||||||
/// This should contain PEM-format certificates
|
/// This should contain PEM-format certificates
|
||||||
/// in the right order (the first certificate should
|
/// in the right order (the first certificate should
|
||||||
/// certify KEYFILE, the last should be a root CA).
|
/// certify KEYFILE, the last should be a root CA).
|
||||||
#[structopt(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))]
|
#[clap(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))]
|
||||||
pub ssl_cert_path: Option<PathBuf>,
|
pub ssl_cert_path: Option<PathBuf>,
|
||||||
|
|
||||||
/// Read private key from KEYFILE. This should be a RSA
|
/// Read private key from KEYFILE. This should be a RSA
|
||||||
/// private key or PKCS8-encoded private key, in PEM format.
|
/// private key or PKCS8-encoded private key, in PEM format.
|
||||||
#[structopt(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))]
|
#[clap(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))]
|
||||||
pub ssl_key_path: Option<PathBuf>,
|
pub ssl_key_path: Option<PathBuf>,
|
||||||
|
|
||||||
/// Enable client authentication, and accept certificates
|
/// Enable client authentication, and accept certificates
|
||||||
/// signed by those roots provided in CERTFILE.
|
/// signed by those roots provided in CERTFILE.
|
||||||
#[structopt(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))]
|
#[clap(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))]
|
||||||
pub ssl_auth_path: Option<PathBuf>,
|
pub ssl_auth_path: Option<PathBuf>,
|
||||||
|
|
||||||
/// Read DER-encoded OCSP response from OCSPFILE and staple to certificate.
|
/// Read DER-encoded OCSP response from OCSPFILE and staple to certificate.
|
||||||
/// Optional
|
/// Optional
|
||||||
#[structopt(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))]
|
#[clap(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))]
|
||||||
pub ssl_ocsp_path: Option<PathBuf>,
|
pub ssl_ocsp_path: Option<PathBuf>,
|
||||||
|
|
||||||
/// Send a fatal alert if the client does not complete client authentication.
|
/// Send a fatal alert if the client does not complete client authentication.
|
||||||
#[structopt(long, env = "MEILI_SSL_REQUIRE_AUTH")]
|
#[clap(long, env = "MEILI_SSL_REQUIRE_AUTH")]
|
||||||
pub ssl_require_auth: bool,
|
pub ssl_require_auth: bool,
|
||||||
|
|
||||||
/// SSL support session resumption
|
/// SSL support session resumption
|
||||||
#[structopt(long, env = "MEILI_SSL_RESUMPTION")]
|
#[clap(long, env = "MEILI_SSL_RESUMPTION")]
|
||||||
pub ssl_resumption: bool,
|
pub ssl_resumption: bool,
|
||||||
|
|
||||||
/// SSL support tickets.
|
/// SSL support tickets.
|
||||||
#[structopt(long, env = "MEILI_SSL_TICKETS")]
|
#[clap(long, env = "MEILI_SSL_TICKETS")]
|
||||||
pub ssl_tickets: bool,
|
pub ssl_tickets: bool,
|
||||||
|
|
||||||
/// Defines the path of the snapshot file to import.
|
/// Defines the path of the snapshot file to import.
|
||||||
/// This option will, by default, stop the process if a database already exist or if no snapshot exists at
|
/// This option will, by default, stop the process if a database already exist or if no snapshot exists at
|
||||||
/// the given path. If this option is not specified no snapshot is imported.
|
/// the given path. If this option is not specified no snapshot is imported.
|
||||||
#[structopt(long)]
|
#[clap(long)]
|
||||||
pub import_snapshot: Option<PathBuf>,
|
pub import_snapshot: Option<PathBuf>,
|
||||||
|
|
||||||
/// The engine will ignore a missing snapshot and not return an error in such case.
|
/// The engine will ignore a missing snapshot and not return an error in such case.
|
||||||
#[structopt(long, requires = "import-snapshot")]
|
#[clap(long, requires = "import-snapshot")]
|
||||||
pub ignore_missing_snapshot: bool,
|
pub ignore_missing_snapshot: bool,
|
||||||
|
|
||||||
/// The engine will skip snapshot importation and not return an error in such case.
|
/// The engine will skip snapshot importation and not return an error in such case.
|
||||||
#[structopt(long, requires = "import-snapshot")]
|
#[clap(long, requires = "import-snapshot")]
|
||||||
pub ignore_snapshot_if_db_exists: bool,
|
pub ignore_snapshot_if_db_exists: bool,
|
||||||
|
|
||||||
/// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap.
|
/// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap.
|
||||||
#[structopt(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")]
|
#[clap(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")]
|
||||||
pub snapshot_dir: PathBuf,
|
pub snapshot_dir: PathBuf,
|
||||||
|
|
||||||
/// Activate snapshot scheduling.
|
/// Activate snapshot scheduling.
|
||||||
#[structopt(long, env = "MEILI_SCHEDULE_SNAPSHOT")]
|
#[clap(long, env = "MEILI_SCHEDULE_SNAPSHOT")]
|
||||||
pub schedule_snapshot: bool,
|
pub schedule_snapshot: bool,
|
||||||
|
|
||||||
/// Defines time interval, in seconds, between each snapshot creation.
|
/// Defines time interval, in seconds, between each snapshot creation.
|
||||||
#[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h
|
#[clap(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h
|
||||||
pub snapshot_interval_sec: u64,
|
pub snapshot_interval_sec: u64,
|
||||||
|
|
||||||
/// Folder where dumps are created when the dump route is called.
|
/// Folder where dumps are created when the dump route is called.
|
||||||
#[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")]
|
#[clap(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")]
|
||||||
pub dumps_dir: PathBuf,
|
pub dumps_dir: PathBuf,
|
||||||
|
|
||||||
/// Import a dump from the specified path, must be a `.dump` file.
|
/// Import a dump from the specified path, must be a `.dump` file.
|
||||||
#[structopt(long, conflicts_with = "import-snapshot")]
|
#[clap(long, conflicts_with = "import-snapshot")]
|
||||||
pub import_dump: Option<PathBuf>,
|
pub import_dump: Option<PathBuf>,
|
||||||
|
|
||||||
/// Set the log level
|
/// Set the log level
|
||||||
#[structopt(long, env = "MEILI_LOG_LEVEL", default_value = "info")]
|
#[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")]
|
||||||
pub log_level: String,
|
pub log_level: String,
|
||||||
|
|
||||||
#[structopt(skip)]
|
#[clap(skip)]
|
||||||
pub indexer_options: IndexerOpts,
|
pub indexer_options: IndexerOpts,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Opt {
|
impl Opt {
|
||||||
/// Wether analytics should be enabled or not.
|
|
||||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
|
||||||
pub fn analytics(&self) -> bool {
|
|
||||||
match self.no_analytics {
|
|
||||||
None => true,
|
|
||||||
Some(None) => false,
|
|
||||||
Some(Some(disabled)) => !disabled,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_ssl_config(&self) -> anyhow::Result<Option<rustls::ServerConfig>> {
|
pub fn get_ssl_config(&self) -> anyhow::Result<Option<rustls::ServerConfig>> {
|
||||||
if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) {
|
if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) {
|
||||||
let client_auth = match &self.ssl_auth_path {
|
let client_auth = match &self.ssl_auth_path {
|
||||||
|
@ -35,15 +35,12 @@ pub struct Index<'a> {
|
|||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
impl Index<'_> {
|
impl Index<'_> {
|
||||||
pub async fn get(&self) -> (Value, StatusCode) {
|
pub async fn get(&self) -> (Value, StatusCode) {
|
||||||
let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string());
|
let url = format!("/indexes/{}", encode(self.uid.as_ref()));
|
||||||
self.service.get(url).await
|
self.service.get(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn load_test_set(&self) -> u64 {
|
pub async fn load_test_set(&self) -> u64 {
|
||||||
let url = format!(
|
let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
|
||||||
"/indexes/{}/documents",
|
|
||||||
encode(self.uid.as_ref()).to_string()
|
|
||||||
);
|
|
||||||
let (response, code) = self
|
let (response, code) = self
|
||||||
.service
|
.service
|
||||||
.post_str(url, include_str!("../assets/test_set.json"))
|
.post_str(url, include_str!("../assets/test_set.json"))
|
||||||
@ -66,13 +63,13 @@ impl Index<'_> {
|
|||||||
let body = json!({
|
let body = json!({
|
||||||
"primaryKey": primary_key,
|
"primaryKey": primary_key,
|
||||||
});
|
});
|
||||||
let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string());
|
let url = format!("/indexes/{}", encode(self.uid.as_ref()));
|
||||||
|
|
||||||
self.service.put(url, body).await
|
self.service.put(url, body).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete(&self) -> (Value, StatusCode) {
|
pub async fn delete(&self) -> (Value, StatusCode) {
|
||||||
let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string());
|
let url = format!("/indexes/{}", encode(self.uid.as_ref()));
|
||||||
self.service.delete(url).await
|
self.service.delete(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -84,13 +81,10 @@ impl Index<'_> {
|
|||||||
let url = match primary_key {
|
let url = match primary_key {
|
||||||
Some(key) => format!(
|
Some(key) => format!(
|
||||||
"/indexes/{}/documents?primaryKey={}",
|
"/indexes/{}/documents?primaryKey={}",
|
||||||
encode(self.uid.as_ref()).to_string(),
|
encode(self.uid.as_ref()),
|
||||||
key
|
key
|
||||||
),
|
),
|
||||||
None => format!(
|
None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
|
||||||
"/indexes/{}/documents",
|
|
||||||
encode(self.uid.as_ref()).to_string()
|
|
||||||
),
|
|
||||||
};
|
};
|
||||||
self.service.post(url, documents).await
|
self.service.post(url, documents).await
|
||||||
}
|
}
|
||||||
@ -103,13 +97,10 @@ impl Index<'_> {
|
|||||||
let url = match primary_key {
|
let url = match primary_key {
|
||||||
Some(key) => format!(
|
Some(key) => format!(
|
||||||
"/indexes/{}/documents?primaryKey={}",
|
"/indexes/{}/documents?primaryKey={}",
|
||||||
encode(self.uid.as_ref()).to_string(),
|
encode(self.uid.as_ref()),
|
||||||
key
|
key
|
||||||
),
|
),
|
||||||
None => format!(
|
None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
|
||||||
"/indexes/{}/documents",
|
|
||||||
encode(self.uid.as_ref()).to_string()
|
|
||||||
),
|
|
||||||
};
|
};
|
||||||
self.service.put(url, documents).await
|
self.service.put(url, documents).await
|
||||||
}
|
}
|
||||||
@ -145,19 +136,12 @@ impl Index<'_> {
|
|||||||
id: u64,
|
id: u64,
|
||||||
_options: Option<GetDocumentOptions>,
|
_options: Option<GetDocumentOptions>,
|
||||||
) -> (Value, StatusCode) {
|
) -> (Value, StatusCode) {
|
||||||
let url = format!(
|
let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
|
||||||
"/indexes/{}/documents/{}",
|
|
||||||
encode(self.uid.as_ref()).to_string(),
|
|
||||||
id
|
|
||||||
);
|
|
||||||
self.service.get(url).await
|
self.service.get(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
|
pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
|
||||||
let mut url = format!(
|
let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref()));
|
||||||
"/indexes/{}/documents?",
|
|
||||||
encode(self.uid.as_ref()).to_string()
|
|
||||||
);
|
|
||||||
if let Some(limit) = options.limit {
|
if let Some(limit) = options.limit {
|
||||||
url.push_str(&format!("limit={}&", limit));
|
url.push_str(&format!("limit={}&", limit));
|
||||||
}
|
}
|
||||||
@ -177,26 +161,19 @@ impl Index<'_> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
|
pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
|
||||||
let url = format!(
|
let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
|
||||||
"/indexes/{}/documents/{}",
|
|
||||||
encode(self.uid.as_ref()).to_string(),
|
|
||||||
id
|
|
||||||
);
|
|
||||||
self.service.delete(url).await
|
self.service.delete(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
|
pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
|
||||||
let url = format!(
|
let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
|
||||||
"/indexes/{}/documents",
|
|
||||||
encode(self.uid.as_ref()).to_string()
|
|
||||||
);
|
|
||||||
self.service.delete(url).await
|
self.service.delete(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
|
pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
|
||||||
let url = format!(
|
let url = format!(
|
||||||
"/indexes/{}/documents/delete-batch",
|
"/indexes/{}/documents/delete-batch",
|
||||||
encode(self.uid.as_ref()).to_string()
|
encode(self.uid.as_ref())
|
||||||
);
|
);
|
||||||
self.service
|
self.service
|
||||||
.post(url, serde_json::to_value(&ids).unwrap())
|
.post(url, serde_json::to_value(&ids).unwrap())
|
||||||
@ -204,31 +181,22 @@ impl Index<'_> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn settings(&self) -> (Value, StatusCode) {
|
pub async fn settings(&self) -> (Value, StatusCode) {
|
||||||
let url = format!(
|
let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
|
||||||
"/indexes/{}/settings",
|
|
||||||
encode(self.uid.as_ref()).to_string()
|
|
||||||
);
|
|
||||||
self.service.get(url).await
|
self.service.get(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
|
pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
|
||||||
let url = format!(
|
let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
|
||||||
"/indexes/{}/settings",
|
|
||||||
encode(self.uid.as_ref()).to_string()
|
|
||||||
);
|
|
||||||
self.service.post(url, settings).await
|
self.service.post(url, settings).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete_settings(&self) -> (Value, StatusCode) {
|
pub async fn delete_settings(&self) -> (Value, StatusCode) {
|
||||||
let url = format!(
|
let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
|
||||||
"/indexes/{}/settings",
|
|
||||||
encode(self.uid.as_ref()).to_string()
|
|
||||||
);
|
|
||||||
self.service.delete(url).await
|
self.service.delete(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn stats(&self) -> (Value, StatusCode) {
|
pub async fn stats(&self) -> (Value, StatusCode) {
|
||||||
let url = format!("/indexes/{}/stats", encode(self.uid.as_ref()).to_string());
|
let url = format!("/indexes/{}/stats", encode(self.uid.as_ref()));
|
||||||
self.service.get(url).await
|
self.service.get(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -253,17 +221,13 @@ impl Index<'_> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
|
pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
|
||||||
let url = format!("/indexes/{}/search", encode(self.uid.as_ref()).to_string());
|
let url = format!("/indexes/{}/search", encode(self.uid.as_ref()));
|
||||||
self.service.post(url, query).await
|
self.service.post(url, query).await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn search_get(&self, query: Value) -> (Value, StatusCode) {
|
pub async fn search_get(&self, query: Value) -> (Value, StatusCode) {
|
||||||
let params = serde_url_params::to_string(&query).unwrap();
|
let params = serde_url_params::to_string(&query).unwrap();
|
||||||
let url = format!(
|
let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params);
|
||||||
"/indexes/{}/search?{}",
|
|
||||||
encode(self.uid.as_ref()).to_string(),
|
|
||||||
params
|
|
||||||
);
|
|
||||||
self.service.get(url).await
|
self.service.get(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -130,7 +130,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
|
|||||||
master_key: None,
|
master_key: None,
|
||||||
env: "development".to_owned(),
|
env: "development".to_owned(),
|
||||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
||||||
no_analytics: Some(Some(true)),
|
no_analytics: true,
|
||||||
max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
|
max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
|
||||||
max_task_db_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
|
max_task_db_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
|
||||||
http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(),
|
http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(),
|
||||||
|
@ -42,7 +42,7 @@ serde = { version = "1.0.130", features = ["derive"] }
|
|||||||
serde_json = { version = "1.0.67", features = ["preserve_order"] }
|
serde_json = { version = "1.0.67", features = ["preserve_order"] }
|
||||||
siphasher = "0.3.7"
|
siphasher = "0.3.7"
|
||||||
slice-group-by = "0.2.6"
|
slice-group-by = "0.2.6"
|
||||||
structopt = "0.3.23"
|
clap = { version = "3.0", features = ["derive", "env"] }
|
||||||
tar = "0.4.37"
|
tar = "0.4.37"
|
||||||
tempfile = "3.2.0"
|
tempfile = "3.2.0"
|
||||||
thiserror = "1.0.28"
|
thiserror = "1.0.28"
|
||||||
|
@ -28,9 +28,7 @@ impl Index {
|
|||||||
pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
|
pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
|
||||||
// acquire write txn make sure any ongoing write is finished before we start.
|
// acquire write txn make sure any ongoing write is finished before we start.
|
||||||
let txn = self.env.write_txn()?;
|
let txn = self.env.write_txn()?;
|
||||||
let path = path
|
let path = path.as_ref().join(format!("indexes/{}", self.uuid));
|
||||||
.as_ref()
|
|
||||||
.join(format!("indexes/{}", self.uuid.to_string()));
|
|
||||||
|
|
||||||
create_dir_all(&path)?;
|
create_dir_all(&path)?;
|
||||||
|
|
||||||
|
@ -2,19 +2,19 @@ use core::fmt;
|
|||||||
use std::{ops::Deref, str::FromStr};
|
use std::{ops::Deref, str::FromStr};
|
||||||
|
|
||||||
use byte_unit::{Byte, ByteError};
|
use byte_unit::{Byte, ByteError};
|
||||||
|
use clap::Parser;
|
||||||
use milli::CompressionType;
|
use milli::CompressionType;
|
||||||
use structopt::StructOpt;
|
|
||||||
use sysinfo::{RefreshKind, System, SystemExt};
|
use sysinfo::{RefreshKind, System, SystemExt};
|
||||||
|
|
||||||
#[derive(Debug, Clone, StructOpt)]
|
#[derive(Debug, Clone, Parser)]
|
||||||
pub struct IndexerOpts {
|
pub struct IndexerOpts {
|
||||||
/// The amount of documents to skip before printing
|
/// The amount of documents to skip before printing
|
||||||
/// a log regarding the indexing advancement.
|
/// a log regarding the indexing advancement.
|
||||||
#[structopt(long, default_value = "100000")] // 100k
|
#[clap(long, default_value = "100000")] // 100k
|
||||||
pub log_every_n: usize,
|
pub log_every_n: usize,
|
||||||
|
|
||||||
/// Grenad max number of chunks in bytes.
|
/// Grenad max number of chunks in bytes.
|
||||||
#[structopt(long)]
|
#[clap(long)]
|
||||||
pub max_nb_chunks: Option<usize>,
|
pub max_nb_chunks: Option<usize>,
|
||||||
|
|
||||||
/// The maximum amount of memory the indexer will use. It defaults to 2/3
|
/// The maximum amount of memory the indexer will use. It defaults to 2/3
|
||||||
@ -24,22 +24,22 @@ pub struct IndexerOpts {
|
|||||||
/// In case the engine is unable to retrieve the available memory the engine will
|
/// In case the engine is unable to retrieve the available memory the engine will
|
||||||
/// try to use the memory it needs but without real limit, this can lead to
|
/// try to use the memory it needs but without real limit, this can lead to
|
||||||
/// Out-Of-Memory issues and it is recommended to specify the amount of memory to use.
|
/// Out-Of-Memory issues and it is recommended to specify the amount of memory to use.
|
||||||
#[structopt(long, default_value)]
|
#[clap(long, default_value_t)]
|
||||||
pub max_memory: MaxMemory,
|
pub max_memory: MaxMemory,
|
||||||
|
|
||||||
/// The name of the compression algorithm to use when compressing intermediate
|
/// The name of the compression algorithm to use when compressing intermediate
|
||||||
/// Grenad chunks while indexing documents.
|
/// Grenad chunks while indexing documents.
|
||||||
///
|
///
|
||||||
/// Choosing a fast algorithm will make the indexing faster but may consume more memory.
|
/// Choosing a fast algorithm will make the indexing faster but may consume more memory.
|
||||||
#[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
|
#[clap(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
|
||||||
pub chunk_compression_type: CompressionType,
|
pub chunk_compression_type: CompressionType,
|
||||||
|
|
||||||
/// The level of compression of the chosen algorithm.
|
/// The level of compression of the chosen algorithm.
|
||||||
#[structopt(long, requires = "chunk-compression-type")]
|
#[clap(long, requires = "chunk-compression-type")]
|
||||||
pub chunk_compression_level: Option<u32>,
|
pub chunk_compression_level: Option<u32>,
|
||||||
|
|
||||||
/// Number of parallel jobs for indexing, defaults to # of CPUs.
|
/// Number of parallel jobs for indexing, defaults to # of CPUs.
|
||||||
#[structopt(long)]
|
#[clap(long)]
|
||||||
pub indexing_jobs: Option<usize>,
|
pub indexing_jobs: Option<usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user