From b4038597ba363a9c3bdf97e1552a005ca624d9ba Mon Sep 17 00:00:00 2001 From: many Date: Mon, 18 Oct 2021 14:16:35 +0200 Subject: [PATCH 1/2] Keep persisting tmp files in database directory and put non-persisting tmp files in default tmp dir --- meilisearch-http/src/lib.rs | 22 ------------------- meilisearch-http/src/main.rs | 4 ---- .../index_controller/dump_actor/loaders/v2.rs | 9 ++++---- .../src/index_controller/dump_actor/mod.rs | 2 +- .../src/index_controller/snapshot.rs | 2 +- .../src/index_controller/update_file_store.rs | 4 ++-- .../index_controller/updates/store/dump.rs | 2 +- 7 files changed, 10 insertions(+), 35 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index cbb417ab1..7ca967ce2 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -8,7 +8,6 @@ pub mod analytics; pub mod helpers; pub mod option; pub mod routes; -use std::path::Path; use std::time::Duration; use crate::error::MeilisearchHttpError; @@ -75,27 +74,6 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result { meilisearch.build(opt.db_path.clone(), opt.indexer_options.clone()) } -/// Cleans and setup the temporary file folder in the database directory. This must be done after -/// the meilisearch instance has been created, to not interfere with the snapshot and dump loading. -pub fn setup_temp_dir(db_path: impl AsRef) -> anyhow::Result<()> { - // Set the tempfile directory in the current db path, to avoid cross device references. Also - // remove the previous outstanding files found there - // - // TODO: if two processes open the same db, one might delete the other tmpdir. Need to make - // sure that no one is using it before deleting it. - let temp_path = db_path.as_ref().join("tmp"); - // Ignore error if tempdir doesn't exist - let _ = std::fs::remove_dir_all(&temp_path); - std::fs::create_dir_all(&temp_path)?; - if cfg!(windows) { - std::env::set_var("TMP", temp_path); - } else { - std::env::set_var("TMPDIR", temp_path); - } - - Ok(()) -} - pub fn configure_data(config: &mut web::ServiceConfig, data: MeiliSearch, opt: &Opt) { let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize; config diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index 213df42fe..864015dd1 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -46,10 +46,6 @@ async fn main() -> anyhow::Result<()> { let meilisearch = setup_meilisearch(&opt)?; - // Setup the temp directory to be in the db folder. This is important, since temporary file - // don't support to be persisted accross filesystem boundaries. - meilisearch_http::setup_temp_dir(&opt.db_path)?; - #[cfg(all(not(debug_assertions), feature = "analytics"))] if !opt.no_analytics { let analytics_data = meilisearch.clone(); diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs index 35640aaef..0a3ffc86a 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs @@ -43,8 +43,9 @@ pub fn load_dump( patch_settings(settings_path)?; } - let update_path = src.as_ref().join("updates/data.jsonl"); - patch_updates(update_path)?; + let update_dir = src.as_ref().join("updates"); + let update_path = update_dir.join("data.jsonl"); + patch_updates(update_dir, update_path)?; v3::load_dump( meta, @@ -79,8 +80,8 @@ fn patch_settings(path: impl AsRef) -> anyhow::Result<()> { Ok(()) } -fn patch_updates(path: impl AsRef) -> anyhow::Result<()> { - let mut output_update_file = NamedTempFile::new()?; +fn patch_updates(dir: impl AsRef, path: impl AsRef) -> anyhow::Result<()> { + let mut output_update_file = NamedTempFile::new_in(&dir)?; let update_file = File::open(&path)?; let stream = Deserializer::from_reader(update_file).into_iter::(); diff --git a/meilisearch-lib/src/index_controller/dump_actor/mod.rs b/meilisearch-lib/src/index_controller/dump_actor/mod.rs index 70f0f8889..76a21398a 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/mod.rs @@ -253,7 +253,7 @@ where UpdateMsg::dump(&self.update_sender, uuids, temp_dump_path.clone()).await?; let dump_path = tokio::task::spawn_blocking(move || -> Result { - let temp_dump_file = tempfile::NamedTempFile::new()?; + let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; to_tar_gz(temp_dump_path, temp_dump_file.path()) .map_err(|e| DumpActorError::Internal(e.into()))?; diff --git a/meilisearch-lib/src/index_controller/snapshot.rs b/meilisearch-lib/src/index_controller/snapshot.rs index 6a22a285c..c0d409990 100644 --- a/meilisearch-lib/src/index_controller/snapshot.rs +++ b/meilisearch-lib/src/index_controller/snapshot.rs @@ -81,7 +81,7 @@ where .snapshot_path .join(format!("{}.snapshot", self.db_name)); let snapshot_path = spawn_blocking(move || -> anyhow::Result { - let temp_snapshot_file = tempfile::NamedTempFile::new()?; + let temp_snapshot_file = tempfile::NamedTempFile::new_in(&snapshot_dir)?; let temp_snapshot_file_path = temp_snapshot_file.path().to_owned(); crate::compression::to_tar_gz(temp_snapshot_path, temp_snapshot_file_path)?; temp_snapshot_file.persist(&snapshot_path)?; diff --git a/meilisearch-lib/src/index_controller/update_file_store.rs b/meilisearch-lib/src/index_controller/update_file_store.rs index 5ee93c58a..db19c2d4c 100644 --- a/meilisearch-lib/src/index_controller/update_file_store.rs +++ b/meilisearch-lib/src/index_controller/update_file_store.rs @@ -107,7 +107,7 @@ impl UpdateFileStore { /// /// A call to `persist` is needed to persist the file in the database. pub fn new_update(&self) -> Result<(Uuid, UpdateFile)> { - let file = NamedTempFile::new()?; + let file = NamedTempFile::new_in(&self.path)?; let uuid = Uuid::new_v4(); let path = self.path.join(uuid.to_string()); let update_file = UpdateFile { file, path }; @@ -141,7 +141,7 @@ impl UpdateFileStore { dst.push(&uuid_string); let update_file = File::open(update_file_path)?; - let mut dst_file = NamedTempFile::new()?; + let mut dst_file = NamedTempFile::new_in(&dump_path)?; let mut document_reader = DocumentBatchReader::from_reader(update_file)?; let mut document_buffer = Map::new(); diff --git a/meilisearch-lib/src/index_controller/updates/store/dump.rs b/meilisearch-lib/src/index_controller/updates/store/dump.rs index 48e1ec821..a9cd256ef 100644 --- a/meilisearch-lib/src/index_controller/updates/store/dump.rs +++ b/meilisearch-lib/src/index_controller/updates/store/dump.rs @@ -52,7 +52,7 @@ impl UpdateStore { uuids: &HashSet, path: impl AsRef, ) -> Result<()> { - let mut dump_data_file = NamedTempFile::new()?; + let mut dump_data_file = NamedTempFile::new_in(&path)?; self.dump_pending(txn, uuids, &mut dump_data_file, &path)?; self.dump_completed(txn, uuids, &mut dump_data_file)?; From 36ab7b3ebdc8418ca36fd84e2e39decab0a5801a Mon Sep 17 00:00:00 2001 From: many Date: Mon, 18 Oct 2021 14:17:32 +0200 Subject: [PATCH 2/2] Fix small typo --- meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs index 0a3ffc86a..13d770f6a 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs @@ -70,7 +70,7 @@ fn patch_settings(path: impl AsRef) -> anyhow::Result<()> { // We first deserialize the dump meta into a serde_json::Value and change // the custom ranking rules settings from the old format to the new format. if let Some(ranking_rules) = meta.pointer_mut("/settings/rankingRules") { - patch_custon_ranking_rules(ranking_rules); + patch_custom_ranking_rules(ranking_rules); } let mut meta_file = OpenOptions::new().truncate(true).write(true).open(path)?; @@ -105,7 +105,7 @@ fn patch_updates(dir: impl AsRef, path: impl AsRef) -> anyhow::Resul /// /// This is done for compatibility reasons, and to avoid a new dump version, /// since the new syntax was introduced soon after the new dump version. -fn patch_custon_ranking_rules(ranking_rules: &mut Value) { +fn patch_custom_ranking_rules(ranking_rules: &mut Value) { *ranking_rules = match ranking_rules.take() { Value::Array(values) => values .into_iter()