From e049aead163463e8da08363ad440a8ea60c21068 Mon Sep 17 00:00:00 2001 From: many Date: Mon, 12 Oct 2020 10:57:19 +0200 Subject: [PATCH 1/8] improve dump status --- Cargo.lock | 6 +++--- meilisearch-http/src/dump.rs | 33 +++++++++++++++--------------- meilisearch-http/src/error.rs | 36 ++++++++++++++++++++++----------- meilisearch-http/tests/dump.rs | 37 ++++++++++++++++++++++++++++++++-- 4 files changed, 79 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b55c1c8a3..19ef76d70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1898,7 +1898,8 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pest" version = "2.1.3" -source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" dependencies = [ "ucd-trie", ] @@ -1906,8 +1907,7 @@ dependencies = [ [[package]] name = "pest" version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" dependencies = [ "ucd-trie", ] diff --git a/meilisearch-http/src/dump.rs b/meilisearch-http/src/dump.rs index 5c0557e6b..87ed869f1 100644 --- a/meilisearch-http/src/dump.rs +++ b/meilisearch-http/src/dump.rs @@ -13,10 +13,11 @@ use meilisearch_core::settings::Settings; use meilisearch_core::update::{apply_settings_update, apply_documents_addition}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; +use serde_json::json; use tempfile::TempDir; use crate::Data; -use crate::error::Error; +use crate::error::{Error, ResponseError}; use crate::helpers::compression; use crate::routes::index; use crate::routes::index::IndexResponse; @@ -112,7 +113,7 @@ fn import_index_v1( // extract `settings.json` file and import content let settings = settings_from_path(&index_path)?; - let settings = settings.to_update().map_err(|_e| Error::dump_failed())?; + let settings = settings.to_update().map_err(|e| Error::dump_failed(format!("importing settings for index {}; {}", index_uid, e)))?; apply_settings_update(write_txn, &index, settings)?; // create iterator over documents in `documents.jsonl` to make batch importation @@ -199,17 +200,17 @@ pub fn import_dump( #[serde(rename_all = "snake_case")] pub enum DumpStatus { Done, - Processing, - DumpProcessFailed, + InProgress, + Failed, } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Clone)] #[serde(rename_all = "camelCase")] pub struct DumpInfo { pub uid: String, pub status: DumpStatus, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, + #[serde(skip_serializing_if = "Option::is_none", flatten)] + pub error: Option, } impl DumpInfo { @@ -217,15 +218,15 @@ impl DumpInfo { Self { uid, status, error: None } } - pub fn with_error(mut self, error: String) -> Self { - self.status = DumpStatus::DumpProcessFailed; - self.error = Some(error); + pub fn with_error(mut self, error: ResponseError) -> Self { + self.status = DumpStatus::Failed; + self.error = Some(json!(error)); self } pub fn dump_already_in_progress(&self) -> bool { - self.status == DumpStatus::Processing + self.status == DumpStatus::InProgress } pub fn get_current() -> Option { @@ -299,10 +300,10 @@ fn dump_index_documents(data: &web::Data, reader: &MainReader, folder_path /// Write error with a context. fn fail_dump_process(dump_info: DumpInfo, context: &str, error: E) { - let error = format!("Something went wrong during dump process: {}; {}", context, error); + let error_message = format!("{}; {}", context, error); - error!("{}", &error); - dump_info.with_error(error).set_current(); + error!("Something went wrong during dump process: {}", &error_message); + dump_info.with_error(Error::dump_failed(error_message).into()).set_current(); } /// Main function of dump. @@ -395,7 +396,7 @@ fn dump_process(data: web::Data, dumps_folder: PathBuf, dump_info: DumpInf } pub fn init_dump_process(data: &web::Data, dumps_folder: &Path) -> Result { - create_dir_all(dumps_folder).or(Err(Error::dump_failed()))?; + create_dir_all(dumps_folder).map_err(|e| Error::dump_failed(format!("creating temporary directory {}", e)))?; // check if a dump is already in progress if let Some(resume) = DumpInfo::get_current() { @@ -407,7 +408,7 @@ pub fn init_dump_process(data: &web::Data, dumps_folder: &Path) -> Result< // generate a new dump info let info = DumpInfo::new( generate_uid(), - DumpStatus::Processing + DumpStatus::InProgress ); info.set_current(); diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index e3488df8a..4c6834e87 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -5,7 +5,7 @@ use actix_http::ResponseBuilder; use actix_web as aweb; use actix_web::error::{JsonPayloadError, QueryPayloadError}; use actix_web::http::StatusCode; -use serde_json::json; +use serde::ser::{Serialize, Serializer, SerializeStruct}; use meilisearch_error::{ErrorCode, Code}; @@ -54,7 +54,7 @@ pub enum Error { PayloadTooLarge, UnsupportedMediaType, DumpAlreadyInProgress, - DumpProcessFailed, + DumpProcessFailed(String), } impl error::Error for Error {} @@ -81,7 +81,7 @@ impl ErrorCode for Error { PayloadTooLarge => Code::PayloadTooLarge, UnsupportedMediaType => Code::UnsupportedMediaType, DumpAlreadyInProgress => Code::DumpAlreadyInProgress, - DumpProcessFailed => Code::DumpProcessFailed, + DumpProcessFailed(_) => Code::DumpProcessFailed, } } } @@ -189,8 +189,8 @@ impl Error { Error::DumpAlreadyInProgress } - pub fn dump_failed() -> Error { - Error::DumpProcessFailed + pub fn dump_failed(message: String) -> Error { + Error::DumpProcessFailed(message) } } @@ -215,19 +215,31 @@ impl fmt::Display for Error { Self::PayloadTooLarge => f.write_str("Payload too large"), Self::UnsupportedMediaType => f.write_str("Unsupported media type"), Self::DumpAlreadyInProgress => f.write_str("Another dump is already in progress"), - Self::DumpProcessFailed => f.write_str("Dump process failed"), + Self::DumpProcessFailed(message) => write!(f, "Dump process failed: {}", message), } } } +impl Serialize for ResponseError { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let struct_name = "ResponseError"; + let field_count = 4; + + let mut state = serializer.serialize_struct(struct_name, field_count)?; + state.serialize_field("message", &self.to_string())?; + state.serialize_field("errorCode", &self.error_name())?; + state.serialize_field("errorType", &self.error_type())?; + state.serialize_field("errorLink", &self.error_url())?; + state.end() + } +} + impl aweb::error::ResponseError for ResponseError { fn error_response(&self) -> aweb::HttpResponse { - ResponseBuilder::new(self.status_code()).json(json!({ - "message": self.to_string(), - "errorCode": self.error_name(), - "errorType": self.error_type(), - "errorLink": self.error_url(), - })) + ResponseBuilder::new(self.status_code()).json(&self) } fn status_code(&self) -> StatusCode { diff --git a/meilisearch-http/tests/dump.rs b/meilisearch-http/tests/dump.rs index 75803c7cd..5da91e095 100644 --- a/meilisearch-http/tests/dump.rs +++ b/meilisearch-http/tests/dump.rs @@ -101,7 +101,7 @@ async fn trigger_dump_concurently_should_return_conflict() { #[actix_rt::test] #[ignore] -async fn get_dump_status_early_should_return_processing() { +async fn get_dump_status_early_should_return_in_progress() { let mut server = common::Server::test_server().await; @@ -116,7 +116,7 @@ async fn get_dump_status_early_should_return_processing() { let expected = json!({ "uid": dump_uid, - "status": "processing" + "status": "in_progress" }); assert_eq!(status_code, 200); @@ -150,6 +150,39 @@ async fn get_dump_status_should_return_done() { assert_json_eq!(expected.clone(), value.clone(), ordered: false); } +#[actix_rt::test] +#[ignore] +async fn get_dump_status_should_return_error_provoking_it() { + let mut server = common::Server::test_server().await; + + + let (value, status_code) = server.trigger_dump().await; + + // removing destination directory provoking `No such file or directory` error + std::fs::remove_dir(server.data().dumps_folder.clone()).unwrap(); + + assert_eq!(status_code, 202); + + let dump_uid = value["uid"].as_str().unwrap().to_string(); + + let expected = json!({ + "uid": dump_uid.clone(), + "status": "failed", + "message": "Dump process failed: compressing dump; No such file or directory (os error 2)", + "errorCode": "dump_process_failed", + "errorType": "internal_error", + "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed" + }); + + thread::sleep(Duration::from_secs(1)); // wait dump until process end + + let (value, status_code) = server.get_dump_status(&dump_uid).await; + + assert_eq!(status_code, 200); + + assert_json_eq!(expected.clone(), value.clone(), ordered: false); +} + #[actix_rt::test] #[ignore] async fn dump_metadata_should_be_valid() { From 834f3cc192c1deeb52e94730b28339023f70be24 Mon Sep 17 00:00:00 2001 From: many Date: Tue, 13 Oct 2020 11:17:02 +0200 Subject: [PATCH 2/8] rename folder to dir --- meilisearch-http/src/data.rs | 6 +-- meilisearch-http/src/dump.rs | 66 ++++++++++++------------- meilisearch-http/src/main.rs | 4 +- meilisearch-http/src/option.rs | 8 +-- meilisearch-http/src/routes/dump.rs | 10 ++-- meilisearch-http/src/snapshot.rs | 8 +-- meilisearch-http/tests/common.rs | 2 +- meilisearch-http/tests/documents_add.rs | 2 +- meilisearch-http/tests/dump.rs | 22 ++++----- 9 files changed, 64 insertions(+), 64 deletions(-) diff --git a/meilisearch-http/src/data.rs b/meilisearch-http/src/data.rs index b552e7f6e..783c81fd8 100644 --- a/meilisearch-http/src/data.rs +++ b/meilisearch-http/src/data.rs @@ -27,7 +27,7 @@ impl Deref for Data { pub struct DataInner { pub db: Arc, pub db_path: String, - pub dumps_folder: PathBuf, + pub dumps_dir: PathBuf, pub dump_batch_size: usize, pub api_keys: ApiKeys, pub server_pid: u32, @@ -61,7 +61,7 @@ impl ApiKeys { impl Data { pub fn new(opt: Opt) -> Result> { let db_path = opt.db_path.clone(); - let dumps_folder = opt.dumps_folder.clone(); + let dumps_dir = opt.dumps_dir.clone(); let dump_batch_size = opt.dump_batch_size; let server_pid = std::process::id(); @@ -85,7 +85,7 @@ impl Data { let inner_data = DataInner { db: db.clone(), db_path, - dumps_folder, + dumps_dir, dump_batch_size, api_keys, server_pid, diff --git a/meilisearch-http/src/dump.rs b/meilisearch-http/src/dump.rs index 87ed869f1..50d89bdc2 100644 --- a/meilisearch-http/src/dump.rs +++ b/meilisearch-http/src/dump.rs @@ -52,9 +52,9 @@ impl DumpMetadata { } } - /// Extract DumpMetadata from `metadata.json` file present at provided `folder_path` - fn from_path(folder_path: &Path) -> Result { - let path = folder_path.join("metadata.json"); + /// Extract DumpMetadata from `metadata.json` file present at provided `dir_path` + fn from_path(dir_path: &Path) -> Result { + let path = dir_path.join("metadata.json"); let file = File::open(path)?; let reader = std::io::BufReader::new(file); let metadata = serde_json::from_reader(reader)?; @@ -62,9 +62,9 @@ impl DumpMetadata { Ok(metadata) } - /// Write DumpMetadata in `metadata.json` file at provided `folder_path` - fn to_path(&self, folder_path: &Path) -> Result<(), Error> { - let path = folder_path.join("metadata.json"); + /// Write DumpMetadata in `metadata.json` file at provided `dir_path` + fn to_path(&self, dir_path: &Path) -> Result<(), Error> { + let path = dir_path.join("metadata.json"); let file = File::create(path)?; serde_json::to_writer(file, &self)?; @@ -73,9 +73,9 @@ impl DumpMetadata { } } -/// Extract Settings from `settings.json` file present at provided `folder_path` -fn settings_from_path(folder_path: &Path) -> Result { - let path = folder_path.join("settings.json"); +/// Extract Settings from `settings.json` file present at provided `dir_path` +fn settings_from_path(dir_path: &Path) -> Result { + let path = dir_path.join("settings.json"); let file = File::open(path)?; let reader = std::io::BufReader::new(file); let metadata = serde_json::from_reader(reader)?; @@ -83,9 +83,9 @@ fn settings_from_path(folder_path: &Path) -> Result { Ok(metadata) } -/// Write Settings in `settings.json` file at provided `folder_path` -fn settings_to_path(settings: &Settings, folder_path: &Path) -> Result<(), Error> { - let path = folder_path.join("settings.json"); +/// Write Settings in `settings.json` file at provided `dir_path` +fn settings_to_path(settings: &Settings, dir_path: &Path) -> Result<(), Error> { + let path = dir_path.join("settings.json"); let file = File::create(path)?; serde_json::to_writer(file, settings)?; @@ -96,7 +96,7 @@ fn settings_to_path(settings: &Settings, folder_path: &Path) -> Result<(), Error /// Import settings and documents of a dump with version `DumpVersion::V1` in specified index. fn import_index_v1( data: &Data, - dumps_folder: &Path, + dumps_dir: &Path, index_uid: &str, document_batch_size: usize, write_txn: &mut MainWriter, @@ -108,8 +108,8 @@ fn import_index_v1( .open_index(index_uid) .ok_or(Error::index_not_found(index_uid))?; - // index folder path in dump folder - let index_path = &dumps_folder.join(index_uid); + // index dir path in dump dir + let index_path = &dumps_dir.join(index_uid); // extract `settings.json` file and import content let settings = settings_from_path(&index_path)?; @@ -243,29 +243,29 @@ fn generate_uid() -> String { Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() } -/// Infer dumps_folder from dump_uid -pub fn compressed_dumps_folder(dumps_folder: &Path, dump_uid: &str) -> PathBuf { - dumps_folder.join(format!("{}.tar.gz", dump_uid)) +/// Infer dumps_dir from dump_uid +pub fn compressed_dumps_dir(dumps_dir: &Path, dump_uid: &str) -> PathBuf { + dumps_dir.join(format!("{}.tar.gz", dump_uid)) } /// Write metadata in dump -fn dump_metadata(data: &web::Data, folder_path: &Path, indexes: Vec) -> Result<(), Error> { +fn dump_metadata(data: &web::Data, dir_path: &Path, indexes: Vec) -> Result<(), Error> { let (db_major, db_minor, db_patch) = data.db.version(); let metadata = DumpMetadata::new(indexes, format!("{}.{}.{}", db_major, db_minor, db_patch)); - metadata.to_path(folder_path) + metadata.to_path(dir_path) } /// Export settings of provided index in dump -fn dump_index_settings(data: &web::Data, reader: &MainReader, folder_path: &Path, index_uid: &str) -> Result<(), Error> { +fn dump_index_settings(data: &web::Data, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { let settings = crate::routes::setting::get_all_sync(data, reader, index_uid)?; - settings_to_path(&settings, folder_path) + settings_to_path(&settings, dir_path) } /// Export updates of provided index in dump -fn dump_index_updates(data: &web::Data, reader: &UpdateReader, folder_path: &Path, index_uid: &str) -> Result<(), Error> { - let updates_path = folder_path.join("updates.jsonl"); +fn dump_index_updates(data: &web::Data, reader: &UpdateReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { + let updates_path = dir_path.join("updates.jsonl"); let updates = crate::routes::index::get_all_updates_status_sync(data, reader, index_uid)?; let file = File::create(updates_path)?; @@ -279,8 +279,8 @@ fn dump_index_updates(data: &web::Data, reader: &UpdateReader, folder_path } /// Export documents of provided index in dump -fn dump_index_documents(data: &web::Data, reader: &MainReader, folder_path: &Path, index_uid: &str) -> Result<(), Error> { - let documents_path = folder_path.join("documents.jsonl"); +fn dump_index_documents(data: &web::Data, reader: &MainReader, dir_path: &Path, index_uid: &str) -> Result<(), Error> { + let documents_path = dir_path.join("documents.jsonl"); let file = File::create(documents_path)?; let dump_batch_size = data.dump_batch_size; @@ -307,7 +307,7 @@ fn fail_dump_process(dump_info: DumpInfo, context: &str, e } /// Main function of dump. -fn dump_process(data: web::Data, dumps_folder: PathBuf, dump_info: DumpInfo) { +fn dump_process(data: web::Data, dumps_dir: PathBuf, dump_info: DumpInfo) { // open read transaction on Update let update_reader = match data.db.update_read_txn() { Ok(r) => r, @@ -380,8 +380,8 @@ fn dump_process(data: web::Data, dumps_folder: PathBuf, dump_info: DumpInf } } - // compress dump in a file named `{dump_uid}.tar.gz` in `dumps_folder` - if let Err(e) = crate::helpers::compression::to_tar_gz(&tmp_dir_path, &compressed_dumps_folder(&dumps_folder, &dump_info.uid)) { + // compress dump in a file named `{dump_uid}.tar.gz` in `dumps_dir` + if let Err(e) = crate::helpers::compression::to_tar_gz(&tmp_dir_path, &compressed_dumps_dir(&dumps_dir, &dump_info.uid)) { fail_dump_process(dump_info, "compressing dump", e); return ; } @@ -395,8 +395,8 @@ fn dump_process(data: web::Data, dumps_folder: PathBuf, dump_info: DumpInf resume.set_current(); } -pub fn init_dump_process(data: &web::Data, dumps_folder: &Path) -> Result { - create_dir_all(dumps_folder).map_err(|e| Error::dump_failed(format!("creating temporary directory {}", e)))?; +pub fn init_dump_process(data: &web::Data, dumps_dir: &Path) -> Result { + create_dir_all(dumps_dir).map_err(|e| Error::dump_failed(format!("creating temporary directory {}", e)))?; // check if a dump is already in progress if let Some(resume) = DumpInfo::get_current() { @@ -414,11 +414,11 @@ pub fn init_dump_process(data: &web::Data, dumps_folder: &Path) -> Result< info.set_current(); let data = data.clone(); - let dumps_folder = dumps_folder.to_path_buf(); + let dumps_dir = dumps_dir.to_path_buf(); let info_cloned = info.clone(); // run dump process in a new thread thread::spawn(move || - dump_process(data, dumps_folder, info_cloned) + dump_process(data, dumps_dir, info_cloned) ); Ok(info) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index b746bf428..a53094639 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -74,8 +74,8 @@ async fn main() -> Result<(), MainError> { dump::import_dump(&data, path, opt.dump_batch_size)?; } - if let Some(path) = &opt.snapshot_path { - snapshot::schedule_snapshot(data.clone(), &path, opt.snapshot_interval_sec.unwrap_or(86400))?; + if let Some(dir) = &opt.snapshot_dir { + snapshot::schedule_snapshot(data.clone(), &dir, opt.snapshot_interval_sec.unwrap_or(86400))?; } print_launch_resume(&opt, &data); diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index a7e6ae0ac..793cba17e 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -109,16 +109,16 @@ pub struct Opt { pub ignore_snapshot_if_db_exists: bool, /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap. - #[structopt(long, env = "MEILI_SNAPSHOT_PATH")] - pub snapshot_path: Option, + #[structopt(long, env = "MEILI_SNAPSHOT_DIR")] + pub snapshot_dir: Option, /// Defines time interval, in seconds, between each snapshot creation. #[structopt(long, requires = "snapshot-path", env = "MEILI_SNAPSHOT_INTERVAL_SEC")] pub snapshot_interval_sec: Option, /// Folder where dumps are created when the dump route is called. - #[structopt(long, env = "MEILI_DUMPS_FOLDER", default_value = "dumps/")] - pub dumps_folder: PathBuf, + #[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")] + pub dumps_dir: PathBuf, /// Import a dump from the specified path, must be a `.tar.gz` file. #[structopt(long, env = "MEILI_IMPORT_DUMP", conflicts_with = "load-from-snapshot")] diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index c57a61112..97fafdfa8 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -5,7 +5,7 @@ use actix_web::{get, post}; use actix_web::{HttpResponse, web}; use serde::{Deserialize, Serialize}; -use crate::dump::{DumpInfo, DumpStatus, compressed_dumps_folder, init_dump_process}; +use crate::dump::{DumpInfo, DumpStatus, compressed_dumps_dir, init_dump_process}; use crate::Data; use crate::error::{Error, ResponseError}; use crate::helpers::Authentication; @@ -19,8 +19,8 @@ pub fn services(cfg: &mut web::ServiceConfig) { async fn trigger_dump( data: web::Data, ) -> Result { - let dumps_folder = Path::new(&data.dumps_folder); - match init_dump_process(&data, &dumps_folder) { + let dumps_dir = Path::new(&data.dumps_dir); + match init_dump_process(&data, &dumps_dir) { Ok(resume) => Ok(HttpResponse::Accepted().json(resume)), Err(e) => Err(e.into()) } @@ -42,7 +42,7 @@ async fn get_dump_status( data: web::Data, path: web::Path, ) -> Result { - let dumps_folder = Path::new(&data.dumps_folder); + let dumps_dir = Path::new(&data.dumps_dir); let dump_uid = &path.dump_uid; if let Some(resume) = DumpInfo::get_current() { @@ -51,7 +51,7 @@ async fn get_dump_status( } } - if File::open(compressed_dumps_folder(Path::new(dumps_folder), dump_uid)).is_ok() { + if File::open(compressed_dumps_dir(Path::new(dumps_dir), dump_uid)).is_ok() { let resume = DumpInfo::new( dump_uid.into(), DumpStatus::Done diff --git a/meilisearch-http/src/snapshot.rs b/meilisearch-http/src/snapshot.rs index 3fe3492b9..39a4555ae 100644 --- a/meilisearch-http/src/snapshot.rs +++ b/meilisearch-http/src/snapshot.rs @@ -70,10 +70,10 @@ mod tests { let archive_path = test_dir.join("archive.tar.gz"); let file_1_relative = Path::new("file1.txt"); - let subfolder_relative = Path::new("subfolder/"); - let file_2_relative = Path::new("subfolder/file2.txt"); + let subdir_relative = Path::new("subdir/"); + let file_2_relative = Path::new("subdir/file2.txt"); - create_dir_all(src_dir.join(subfolder_relative)).unwrap(); + create_dir_all(src_dir.join(subdir_relative)).unwrap(); fs::File::create(src_dir.join(file_1_relative)).unwrap().write_all(b"Hello_file_1").unwrap(); fs::File::create(src_dir.join(file_2_relative)).unwrap().write_all(b"Hello_file_2").unwrap(); @@ -84,7 +84,7 @@ mod tests { assert!(dest_dir.exists()); assert!(dest_dir.join(file_1_relative).exists()); - assert!(dest_dir.join(subfolder_relative).exists()); + assert!(dest_dir.join(subdir_relative).exists()); assert!(dest_dir.join(file_2_relative).exists()); let contents = fs::read_to_string(dest_dir.join(file_1_relative)).unwrap(); diff --git a/meilisearch-http/tests/common.rs b/meilisearch-http/tests/common.rs index 0cdd3b618..3b875b4d2 100644 --- a/meilisearch-http/tests/common.rs +++ b/meilisearch-http/tests/common.rs @@ -49,7 +49,7 @@ impl Server { let opt = Opt { db_path: tmp_dir.path().join("db").to_str().unwrap().to_string(), - dumps_folder: tmp_dir.path().join("dump"), + dumps_dir: tmp_dir.path().join("dump"), dump_batch_size: 16, http_addr: "127.0.0.1:7700".to_owned(), master_key: None, diff --git a/meilisearch-http/tests/documents_add.rs b/meilisearch-http/tests/documents_add.rs index 382a1ed43..659cfc4ec 100644 --- a/meilisearch-http/tests/documents_add.rs +++ b/meilisearch-http/tests/documents_add.rs @@ -187,7 +187,7 @@ async fn add_document_with_long_field() { "relurl":"/configuration/app/web.html#locations", "section":"Web", "site":"docs", - "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", + "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The dir from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", "title":"Locations", "url":"/configuration/app/web.html#locations" }]); diff --git a/meilisearch-http/tests/dump.rs b/meilisearch-http/tests/dump.rs index 5da91e095..70b6b08d4 100644 --- a/meilisearch-http/tests/dump.rs +++ b/meilisearch-http/tests/dump.rs @@ -159,7 +159,7 @@ async fn get_dump_status_should_return_error_provoking_it() { let (value, status_code) = server.trigger_dump().await; // removing destination directory provoking `No such file or directory` error - std::fs::remove_dir(server.data().dumps_folder.clone()).unwrap(); + std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap(); assert_eq!(status_code, 202); @@ -197,11 +197,11 @@ async fn dump_metadata_should_be_valid() { let uid = trigger_and_wait_dump(&mut server).await; - let dumps_folder = Path::new(&server.data().dumps_folder); + let dumps_dir = Path::new(&server.data().dumps_dir); let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); - compression::from_tar_gz(&dumps_folder.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); + compression::from_tar_gz(&dumps_dir.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("metadata.json")).unwrap(); let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap(); @@ -238,9 +238,9 @@ async fn dump_gzip_should_have_been_created() { let dump_uid = trigger_and_wait_dump(&mut server).await; - let dumps_folder = Path::new(&server.data().dumps_folder); + let dumps_dir = Path::new(&server.data().dumps_dir); - let compressed_path = dumps_folder.join(format!("{}.tar.gz", dump_uid)); + let compressed_path = dumps_dir.join(format!("{}.tar.gz", dump_uid)); assert!(File::open(compressed_path).is_ok()); } @@ -312,11 +312,11 @@ async fn dump_index_settings_should_be_valid() { let uid = trigger_and_wait_dump(&mut server).await; - let dumps_folder = Path::new(&server.data().dumps_folder); + let dumps_dir = Path::new(&server.data().dumps_dir); let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); - compression::from_tar_gz(&dumps_folder.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); + compression::from_tar_gz(&dumps_dir.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap(); let settings: serde_json::Value = serde_json::from_reader(file).unwrap(); @@ -336,11 +336,11 @@ async fn dump_index_documents_should_be_valid() { let uid = trigger_and_wait_dump(&mut server).await; - let dumps_folder = Path::new(&server.data().dumps_folder); + let dumps_dir = Path::new(&server.data().dumps_dir); let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); - compression::from_tar_gz(&dumps_folder.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); + compression::from_tar_gz(&dumps_dir.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap(); let documents = read_all_jsonline(file); @@ -360,11 +360,11 @@ async fn dump_index_updates_should_be_valid() { let uid = trigger_and_wait_dump(&mut server).await; - let dumps_folder = Path::new(&server.data().dumps_folder); + let dumps_dir = Path::new(&server.data().dumps_dir); let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); - compression::from_tar_gz(&dumps_folder.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); + compression::from_tar_gz(&dumps_dir.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap(); let mut updates = read_all_jsonline(file); From f1925b8f71cce80eed61c0fd7fdc2954d592e5d8 Mon Sep 17 00:00:00 2001 From: many Date: Thu, 15 Oct 2020 12:06:11 +0200 Subject: [PATCH 3/8] fix #1009 --- meilisearch-http/src/main.rs | 6 +++--- meilisearch-http/src/option.rs | 18 +++++++++++------- meilisearch-http/src/snapshot.rs | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index a53094639..563f75d2f 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -52,7 +52,7 @@ async fn main() -> Result<(), MainError> { _ => unreachable!(), } - if let Some(path) = &opt.load_from_snapshot { + if let Some(path) = &opt.import_snapshot { snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; } @@ -74,8 +74,8 @@ async fn main() -> Result<(), MainError> { dump::import_dump(&data, path, opt.dump_batch_size)?; } - if let Some(dir) = &opt.snapshot_dir { - snapshot::schedule_snapshot(data.clone(), &dir, opt.snapshot_interval_sec.unwrap_or(86400))?; + if opt.schedule_snapshot { + snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; } print_launch_resume(&opt, &data); diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 793cba17e..58e88633a 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -97,23 +97,27 @@ pub struct Opt { /// Defines the path of the snapshot file to import. /// This option will, by default, stop the process if a database already exist or if no snapshot exists at /// the given path. If this option is not specified no snapshot is imported. - #[structopt(long, env = "MEILI_LOAD_FROM_SNAPSHOT")] - pub load_from_snapshot: Option, + #[structopt(long, env = "MEILI_IMPORT_SNAPSHOT")] + pub import_snapshot: Option, /// The engine will ignore a missing snapshot and not return an error in such case. - #[structopt(long, requires = "load-from-snapshot", env = "MEILI_IGNORE_MISSING_SNAPSHOT")] + #[structopt(long, requires = "import-snapshot", env = "MEILI_IGNORE_MISSING_SNAPSHOT")] pub ignore_missing_snapshot: bool, /// The engine will skip snapshot importation and not return an error in such case. - #[structopt(long, requires = "load-from-snapshot", env = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS")] + #[structopt(long, requires = "import-snapshot", env = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS")] pub ignore_snapshot_if_db_exists: bool, /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap. - #[structopt(long, env = "MEILI_SNAPSHOT_DIR")] - pub snapshot_dir: Option, + #[structopt(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")] + pub snapshot_dir: PathBuf, + + /// Activate snapshot scheduling. + #[structopt(long, env = "MEILI_SCHEDULE_SNAPSHOT")] + pub schedule_snapshot: bool, /// Defines time interval, in seconds, between each snapshot creation. - #[structopt(long, requires = "snapshot-path", env = "MEILI_SNAPSHOT_INTERVAL_SEC")] + #[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC")] pub snapshot_interval_sec: Option, /// Folder where dumps are created when the dump route is called. diff --git a/meilisearch-http/src/snapshot.rs b/meilisearch-http/src/snapshot.rs index 39a4555ae..c391dbdfa 100644 --- a/meilisearch-http/src/snapshot.rs +++ b/meilisearch-http/src/snapshot.rs @@ -20,7 +20,7 @@ pub fn load_snapshot( if !db_path.exists() && snapshot_path.exists() { compression::from_tar_gz(snapshot_path, db_path) } else if db_path.exists() && !ignore_snapshot_if_db_exists { - Err(Error::Internal(format!("database already exists at {:?}", db_path))) + Err(Error::Internal(format!("database already exists at {:?}, try to delete it or rename it", db_path))) } else if !snapshot_path.exists() && !ignore_missing_snapshot { Err(Error::Internal(format!("snapshot doesn't exist at {:?}", snapshot_path))) } else { From e6033e174d4dbb925fb3cb4532630a47b6b10f73 Mon Sep 17 00:00:00 2001 From: many Date: Thu, 15 Oct 2020 12:18:15 +0200 Subject: [PATCH 4/8] fix #1010 --- meilisearch-http/src/dump.rs | 4 ++-- meilisearch-http/src/snapshot.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/meilisearch-http/src/dump.rs b/meilisearch-http/src/dump.rs index 50d89bdc2..468dbf640 100644 --- a/meilisearch-http/src/dump.rs +++ b/meilisearch-http/src/dump.rs @@ -245,7 +245,7 @@ fn generate_uid() -> String { /// Infer dumps_dir from dump_uid pub fn compressed_dumps_dir(dumps_dir: &Path, dump_uid: &str) -> PathBuf { - dumps_dir.join(format!("{}.tar.gz", dump_uid)) + dumps_dir.join(format!("{}.dump", dump_uid)) } /// Write metadata in dump @@ -380,7 +380,7 @@ fn dump_process(data: web::Data, dumps_dir: PathBuf, dump_info: DumpInfo) } } - // compress dump in a file named `{dump_uid}.tar.gz` in `dumps_dir` + // compress dump in a file named `{dump_uid}.dump` in `dumps_dir` if let Err(e) = crate::helpers::compression::to_tar_gz(&tmp_dir_path, &compressed_dumps_dir(&dumps_dir, &dump_info.uid)) { fail_dump_process(dump_info, "compressing dump", e); return ; diff --git a/meilisearch-http/src/snapshot.rs b/meilisearch-http/src/snapshot.rs index c391dbdfa..980d491d0 100644 --- a/meilisearch-http/src/snapshot.rs +++ b/meilisearch-http/src/snapshot.rs @@ -42,7 +42,7 @@ pub fn schedule_snapshot(data: Data, snapshot_dir: &Path, time_gap_s: u64) -> Re } let db_name = Path::new(&data.db_path).file_name().ok_or_else(|| Error::Internal("invalid database name".to_string()))?; create_dir_all(snapshot_dir)?; - let snapshot_path = snapshot_dir.join(format!("{}.tar.gz", db_name.to_str().unwrap_or("data.ms"))); + let snapshot_path = snapshot_dir.join(format!("{}.snapshot", db_name.to_str().unwrap_or("data.ms"))); thread::spawn(move || loop { thread::sleep(Duration::from_secs(time_gap_s)); @@ -67,7 +67,7 @@ mod tests { let test_dir = tempdir.path(); let src_dir = test_dir.join("src"); let dest_dir = test_dir.join("complex/destination/path/"); - let archive_path = test_dir.join("archive.tar.gz"); + let archive_path = test_dir.join("archive.snapshot"); let file_1_relative = Path::new("file1.txt"); let subdir_relative = Path::new("subdir/"); From 1eace79f7788bb27f5c63cfc674dde3e51be7d8a Mon Sep 17 00:00:00 2001 From: many Date: Thu, 15 Oct 2020 13:38:29 +0200 Subject: [PATCH 5/8] change error message to be absolute --- meilisearch-http/src/snapshot.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/src/snapshot.rs b/meilisearch-http/src/snapshot.rs index 980d491d0..0351aa26f 100644 --- a/meilisearch-http/src/snapshot.rs +++ b/meilisearch-http/src/snapshot.rs @@ -20,9 +20,9 @@ pub fn load_snapshot( if !db_path.exists() && snapshot_path.exists() { compression::from_tar_gz(snapshot_path, db_path) } else if db_path.exists() && !ignore_snapshot_if_db_exists { - Err(Error::Internal(format!("database already exists at {:?}, try to delete it or rename it", db_path))) + Err(Error::Internal(format!("database already exists at {:?}, try to delete it or rename it", db_path.canonicalize().unwrap_or(db_path.into())))) } else if !snapshot_path.exists() && !ignore_missing_snapshot { - Err(Error::Internal(format!("snapshot doesn't exist at {:?}", snapshot_path))) + Err(Error::Internal(format!("snapshot doesn't exist at {:?}", snapshot_path.canonicalize().unwrap_or(snapshot_path.into())))) } else { Ok(()) } From 10dace305d7721d4d40a6de8539770a9a03f2828 Mon Sep 17 00:00:00 2001 From: many Date: Thu, 15 Oct 2020 16:11:02 +0200 Subject: [PATCH 6/8] snapshot at start --- meilisearch-http/src/snapshot.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/snapshot.rs b/meilisearch-http/src/snapshot.rs index 0351aa26f..520044f84 100644 --- a/meilisearch-http/src/snapshot.rs +++ b/meilisearch-http/src/snapshot.rs @@ -45,10 +45,10 @@ pub fn schedule_snapshot(data: Data, snapshot_dir: &Path, time_gap_s: u64) -> Re let snapshot_path = snapshot_dir.join(format!("{}.snapshot", db_name.to_str().unwrap_or("data.ms"))); thread::spawn(move || loop { - thread::sleep(Duration::from_secs(time_gap_s)); if let Err(e) = create_snapshot(&data, &snapshot_path) { error!("Unsuccessful snapshot creation: {}", e); } + thread::sleep(Duration::from_secs(time_gap_s)); }); Ok(()) From a0eafea200bc2aa7defbe2cb8ca129f273b1b9ea Mon Sep 17 00:00:00 2001 From: many Date: Tue, 20 Oct 2020 16:37:12 +0200 Subject: [PATCH 7/8] fix tests --- meilisearch-http/src/option.rs | 8 ++++---- meilisearch-http/tests/dump.rs | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 58e88633a..e1d74fd63 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -97,15 +97,15 @@ pub struct Opt { /// Defines the path of the snapshot file to import. /// This option will, by default, stop the process if a database already exist or if no snapshot exists at /// the given path. If this option is not specified no snapshot is imported. - #[structopt(long, env = "MEILI_IMPORT_SNAPSHOT")] + #[structopt(long)] pub import_snapshot: Option, /// The engine will ignore a missing snapshot and not return an error in such case. - #[structopt(long, requires = "import-snapshot", env = "MEILI_IGNORE_MISSING_SNAPSHOT")] + #[structopt(long, requires = "import-snapshot")] pub ignore_missing_snapshot: bool, /// The engine will skip snapshot importation and not return an error in such case. - #[structopt(long, requires = "import-snapshot", env = "MEILI_IGNORE_SNAPSHOT_IF_DB_EXISTS")] + #[structopt(long, requires = "import-snapshot")] pub ignore_snapshot_if_db_exists: bool, /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap. @@ -125,7 +125,7 @@ pub struct Opt { pub dumps_dir: PathBuf, /// Import a dump from the specified path, must be a `.tar.gz` file. - #[structopt(long, env = "MEILI_IMPORT_DUMP", conflicts_with = "load-from-snapshot")] + #[structopt(long, conflicts_with = "import-snapshot")] pub import_dump: Option, /// The batch size used in the importation process, the bigger it is the faster the dump is created. diff --git a/meilisearch-http/tests/dump.rs b/meilisearch-http/tests/dump.rs index 70b6b08d4..0c2a2b696 100644 --- a/meilisearch-http/tests/dump.rs +++ b/meilisearch-http/tests/dump.rs @@ -201,7 +201,7 @@ async fn dump_metadata_should_be_valid() { let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); - compression::from_tar_gz(&dumps_dir.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("metadata.json")).unwrap(); let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap(); @@ -240,7 +240,7 @@ async fn dump_gzip_should_have_been_created() { let dump_uid = trigger_and_wait_dump(&mut server).await; let dumps_dir = Path::new(&server.data().dumps_dir); - let compressed_path = dumps_dir.join(format!("{}.tar.gz", dump_uid)); + let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid)); assert!(File::open(compressed_path).is_ok()); } @@ -316,7 +316,7 @@ async fn dump_index_settings_should_be_valid() { let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); - compression::from_tar_gz(&dumps_dir.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap(); let settings: serde_json::Value = serde_json::from_reader(file).unwrap(); @@ -340,7 +340,7 @@ async fn dump_index_documents_should_be_valid() { let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); - compression::from_tar_gz(&dumps_dir.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap(); let documents = read_all_jsonline(file); @@ -364,7 +364,7 @@ async fn dump_index_updates_should_be_valid() { let tmp_dir = TempDir::new().unwrap(); let tmp_dir_path = tmp_dir.path(); - compression::from_tar_gz(&dumps_dir.join(&format!("{}.tar.gz", uid)), tmp_dir_path).unwrap(); + compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap(); let mut updates = read_all_jsonline(file); From 53b1483e71b86340d7ecf723d0c658f557256ad2 Mon Sep 17 00:00:00 2001 From: many Date: Thu, 22 Oct 2020 15:29:39 +0200 Subject: [PATCH 8/8] fix pr comments --- Cargo.lock | 6 +- meilisearch-http/src/error.rs | 90 ++++++++++++------------- meilisearch-http/tests/documents_add.rs | 2 +- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19ef76d70..b55c1c8a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1898,8 +1898,7 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pest" version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" dependencies = [ "ucd-trie", ] @@ -1907,7 +1906,8 @@ dependencies = [ [[package]] name = "pest" version = "2.1.3" -source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" dependencies = [ "ucd-trie", ] diff --git a/meilisearch-http/src/error.rs b/meilisearch-http/src/error.rs index 4c6834e87..de3e82dac 100644 --- a/meilisearch-http/src/error.rs +++ b/meilisearch-http/src/error.rs @@ -34,6 +34,51 @@ impl From for ResponseError { } } +impl From for ResponseError { + fn from(err: meilisearch_core::Error) -> ResponseError { + ResponseError { inner: Box::new(err) } + } +} + +impl From for ResponseError { + fn from(err: meilisearch_schema::Error) -> ResponseError { + ResponseError { inner: Box::new(err) } + } +} + +impl From for ResponseError { + fn from(err: FacetCountError) -> ResponseError { + ResponseError { inner: Box::new(err) } + } +} + +impl Serialize for ResponseError { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let struct_name = "ResponseError"; + let field_count = 4; + + let mut state = serializer.serialize_struct(struct_name, field_count)?; + state.serialize_field("message", &self.to_string())?; + state.serialize_field("errorCode", &self.error_name())?; + state.serialize_field("errorType", &self.error_type())?; + state.serialize_field("errorLink", &self.error_url())?; + state.end() + } +} + +impl aweb::error::ResponseError for ResponseError { + fn error_response(&self) -> aweb::HttpResponse { + ResponseBuilder::new(self.status_code()).json(&self) + } + + fn status_code(&self) -> StatusCode { + self.http_status() + } +} + #[derive(Debug)] pub enum Error { BadParameter(String, String), @@ -220,51 +265,12 @@ impl fmt::Display for Error { } } -impl Serialize for ResponseError { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let struct_name = "ResponseError"; - let field_count = 4; - - let mut state = serializer.serialize_struct(struct_name, field_count)?; - state.serialize_field("message", &self.to_string())?; - state.serialize_field("errorCode", &self.error_name())?; - state.serialize_field("errorType", &self.error_type())?; - state.serialize_field("errorLink", &self.error_url())?; - state.end() - } -} - -impl aweb::error::ResponseError for ResponseError { - fn error_response(&self) -> aweb::HttpResponse { - ResponseBuilder::new(self.status_code()).json(&self) - } - - fn status_code(&self) -> StatusCode { - self.http_status() - } -} - impl From for Error { fn from(err: std::io::Error) -> Error { Error::Internal(err.to_string()) } } -impl From for ResponseError { - fn from(err: meilisearch_core::Error) -> ResponseError { - ResponseError { inner: Box::new(err) } - } -} - -impl From for ResponseError { - fn from(err: meilisearch_schema::Error) -> ResponseError { - ResponseError { inner: Box::new(err) } - } -} - impl From for Error { fn from(err: actix_http::Error) -> Error { Error::Internal(err.to_string()) @@ -283,12 +289,6 @@ impl From for Error { } } -impl From for ResponseError { - fn from(err: FacetCountError) -> ResponseError { - ResponseError { inner: Box::new(err) } - } -} - impl From for Error { fn from(err: JsonPayloadError) -> Error { match err { diff --git a/meilisearch-http/tests/documents_add.rs b/meilisearch-http/tests/documents_add.rs index 659cfc4ec..382a1ed43 100644 --- a/meilisearch-http/tests/documents_add.rs +++ b/meilisearch-http/tests/documents_add.rs @@ -187,7 +187,7 @@ async fn add_document_with_long_field() { "relurl":"/configuration/app/web.html#locations", "section":"Web", "site":"docs", - "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The dir from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", + "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", "title":"Locations", "url":"/configuration/app/web.html#locations" }]);