From c47369839bc73b4262b1aa79fb430cc462b65812 Mon Sep 17 00:00:00 2001 From: Marin Postma Date: Thu, 27 May 2021 10:51:19 +0200 Subject: [PATCH] dump meta --- .../src/index_controller/dump_actor/actor.rs | 123 ++++++++++-------- .../index_controller/dump_actor/loaders/v2.rs | 9 ++ .../src/index_controller/dump_actor/mod.rs | 4 + 3 files changed, 82 insertions(+), 54 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/actor.rs b/meilisearch-http/src/index_controller/dump_actor/actor.rs index 31378f89c..1abceef47 100644 --- a/meilisearch-http/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-http/src/index_controller/dump_actor/actor.rs @@ -1,17 +1,17 @@ use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus}; -use crate::helpers::compression; +use crate::{helpers::compression, index_controller::dump_actor::Metadata}; use crate::index_controller::{update_actor, uuid_resolver}; use async_stream::stream; use chrono::Utc; use futures::stream::StreamExt; use log::{error, info}; -use std::{ - path::{Path, PathBuf}, - sync::Arc, -}; +use update_actor::UpdateActorHandle; +use uuid_resolver::UuidResolverHandle; +use std::{fs::File, path::{Path, PathBuf}, sync::Arc}; use tokio::{fs::create_dir_all, sync::{mpsc, oneshot, RwLock}}; pub const CONCURRENT_DUMP_MSG: usize = 10; +const META_FILE_NAME: &'static str = "metadata.json"; pub struct DumpActor { inbox: Option>, @@ -19,8 +19,8 @@ pub struct DumpActor { update: Update, dump_path: PathBuf, dump_info: Arc>>, - _update_db_size: u64, - _index_db_size: u64, + update_db_size: u64, + index_db_size: u64, } /// Generate uid from creation date @@ -30,16 +30,16 @@ fn generate_uid() -> String { impl DumpActor where - UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, - Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, + UuidResolver: UuidResolverHandle + Send + Sync + Clone + 'static, + Update: UpdateActorHandle + Send + Sync + Clone + 'static, { pub fn new( inbox: mpsc::Receiver, uuid_resolver: UuidResolver, update: Update, dump_path: impl AsRef, - _index_db_size: u64, - _update_db_size: u64, + index_db_size: u64, + update_db_size: u64, ) -> Self { Self { inbox: Some(inbox), @@ -47,8 +47,8 @@ where update, dump_path: dump_path.as_ref().into(), dump_info: Arc::new(RwLock::new(None)), - _index_db_size, - _update_db_size, + index_db_size, + update_db_size, } } @@ -103,13 +103,16 @@ where let dump_info = self.dump_info.clone(); - let task_result = tokio::task::spawn(perform_dump( - self.dump_path.clone(), - self.uuid_resolver.clone(), - self.update.clone(), - uid.clone(), - )) - .await; + let task = DumpTask { + path: self.dump_path.clone(), + uuid_resolver: self.uuid_resolver.clone(), + update_handle: self.update.clone(), + uid: uid.clone(), + update_db_size: self.update_db_size, + index_db_size: self.index_db_size, + }; + + let task_result = tokio::task::spawn(task.run()).await; match task_result { Ok(Ok(())) => { @@ -152,42 +155,54 @@ where }) ) } + } -async fn perform_dump( +struct DumpTask { path: PathBuf, - uuid_resolver: UuidResolver, - update_handle: Update, + uuid_resolver: U, + update_handle: P, uid: String, -) -> anyhow::Result<()> -where - UuidResolver: uuid_resolver::UuidResolverHandle + Send + Sync + Clone + 'static, - Update: update_actor::UpdateActorHandle + Send + Sync + Clone + 'static, -{ - info!("Performing dump."); - - create_dir_all(&path).await?; - - let path_clone = path.clone(); - let temp_dump_dir = tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??; - let temp_dump_path = temp_dump_dir.path().to_owned(); - - let uuids = uuid_resolver.dump(temp_dump_path.clone()).await?; - - update_handle.dump(uuids, temp_dump_path.clone()).await?; - - let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(&path)?; - compression::to_tar_gz(temp_dump_path, temp_dump_file.path())?; - - let dump_path = path.join(format!("{}.dump", uid)); - temp_dump_file.persist(&dump_path)?; - - Ok(dump_path) - }) - .await??; - - info!("Created dump in {:?}.", dump_path); - - Ok(()) + update_db_size: u64, + index_db_size: u64, +} + +impl DumpTask +where + U: UuidResolverHandle + Send + Sync + Clone + 'static, + P: UpdateActorHandle + Send + Sync + Clone + 'static, +{ + async fn run(self) -> anyhow::Result<()> { + info!("Performing dump."); + + create_dir_all(&self.path).await?; + + let path_clone = self.path.clone(); + let temp_dump_dir = tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let meta = Metadata::new_v2(self.index_db_size, self.update_db_size); + let meta_path = temp_dump_path.join(META_FILE_NAME); + let mut meta_file = File::create(&meta_path)?; + serde_json::to_writer(&mut meta_file, &meta)?; + + let uuids = self.uuid_resolver.dump(temp_dump_path.clone()).await?; + + self.update_handle.dump(uuids, temp_dump_path.clone()).await?; + + let dump_path = tokio::task::spawn_blocking(move || -> anyhow::Result { + let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; + compression::to_tar_gz(temp_dump_path, temp_dump_file.path())?; + + let dump_path = self.path.join(format!("{}.dump", self.uid)); + temp_dump_file.persist(&dump_path)?; + + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(()) + } } diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs index ab4aa8cff..b9f89ebbf 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs @@ -16,6 +16,15 @@ pub struct MetadataV2 { } impl MetadataV2 { + pub fn new(index_db_size: u64, update_db_size: u64) -> Self { + Self { + db_version: env!("CARGO_PKG_VERSION").to_string(), + index_db_size, + update_db_size, + dump_date: Utc::now(), + } + } + pub fn load_dump( self, src: impl AsRef, diff --git a/meilisearch-http/src/index_controller/dump_actor/mod.rs b/meilisearch-http/src/index_controller/dump_actor/mod.rs index 6d661d75c..b54783f75 100644 --- a/meilisearch-http/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-http/src/index_controller/dump_actor/mod.rs @@ -62,6 +62,10 @@ pub enum Metadata { } impl Metadata { + pub fn new_v2(index_db_size: u64, update_db_size: u64) -> Self { + let meta = MetadataV2::new(index_db_size, update_db_size); + Self::V2 { meta } + } /// Extract Metadata from `metadata.json` file present at provided `dir_path` fn from_path(dir_path: &Path) -> anyhow::Result { let path = dir_path.join("metadata.json");