snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 20:15:07 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

refactorize the dump and snapshot

Browse Source
This commit is contained in:
Tamo 2021-10-26 13:02:40 +02:00 committed by marin postma
parent 87a8bf5e96
commit c752c14c46
No known key found for this signature in database
GPG Key ID: 6088B7721C3E39F9
6 changed files with 28 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
meilisearch-lib/src/analytics.rs
View File

@ -1,17 +1,8 @@
use std::{fs, path::Path};
/// To load a dump we get the user id from the source directory;
/// If there was a user-id, write it to the new destination if not ignore the error
pub fn load_dump(src: &Path, dst: &Path) {
/// Copy the `user-id` contained in one db to another. Ignore all errors.
pub fn copy_user_id(src: &Path, dst: &Path) {
if let Ok(user_id) = fs::read_to_string(src.join("user-id")) {
let _ = fs::write(dst.join("user-id"), &user_id);
}
}
/// To load a dump we get the user id either from the source directory;
/// If there was a user-id, write it to the new destination if not ignore the error
pub fn write_dump(src: &Path, dst: &Path) {
if let Ok(user_id) = fs::read_to_string(src) {
let _ = fs::write(dst, &user_id);
}
}

4
meilisearch-lib/src/index_controller/dump_actor/actor.rs
View File

@ -121,8 +121,8 @@ where
ret.send(Ok(info)).expect("Dump actor is dead");
let task = DumpTask {
path: self.dump_path.clone(),
analytics_path: self.analytics_path.clone(),
dump_path: self.dump_path.clone(),
db_path: self.analytics_path.clone(),
index_resolver: self.index_resolver.clone(),
update_sender: self.update.clone(),
uid: uid.clone(),

2
meilisearch-lib/src/index_controller/dump_actor/loaders/v3.rs
View File

@ -25,7 +25,7 @@ pub fn load_dump(
IndexResolver::load_dump(src.as_ref(), &dst, index_db_size, indexing_options)?;
UpdateFileStore::load_dump(src.as_ref(), &dst)?;
UpdateStore::load_dump(&src, &dst, update_db_size)?;
analytics::load_dump(src.as_ref(), dst.as_ref());
analytics::copy_user_id(src.as_ref(), dst.as_ref());
info!("Loading indexes.");

20
meilisearch-lib/src/index_controller/dump_actor/mod.rs
View File

@ -223,8 +223,8 @@ pub fn load_dump(
}
struct DumpTask<U, I> {
path: PathBuf,
analytics_path: PathBuf,
dump_path: PathBuf,
db_path: PathBuf,
index_resolver: Arc<IndexResolver<U, I>>,
update_sender: UpdateSender,
uid: String,
@ -240,7 +240,7 @@ where
async fn run(self) -> Result<()> {
trace!("Performing dump.");
create_dir_all(&self.path).await?;
create_dir_all(&self.dump_path).await?;
let temp_dump_dir = tokio::task::spawn_blocking(tempfile::TempDir::new).await??;
let temp_dump_path = temp_dump_dir.path().to_owned();
@ -249,7 +249,7 @@ where
let meta_path = temp_dump_path.join(META_FILE_NAME);
let mut meta_file = File::create(&meta_path)?;
serde_json::to_writer(&mut meta_file, &meta)?;
analytics::write_dump(&self.analytics_path, &temp_dump_path.join("user-id"));
analytics::copy_user_id(&self.db_path, &temp_dump_path);
create_dir_all(&temp_dump_path.join("indexes")).await?;
let uuids = self.index_resolver.dump(temp_dump_path.clone()).await?;
@ -257,11 +257,11 @@ where
UpdateMsg::dump(&self.update_sender, uuids, temp_dump_path.clone()).await?;
let dump_path = tokio::task::spawn_blocking(move || -> Result<PathBuf> {
let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?;
let temp_dump_file = tempfile::NamedTempFile::new_in(&self.dump_path)?;
to_tar_gz(temp_dump_path, temp_dump_file.path())
.map_err(|e| DumpActorError::Internal(e.into()))?;
let dump_path = self.path.join(self.uid).with_extension("dump");
let dump_path = self.dump_path.join(self.uid).with_extension("dump");
temp_dump_file.persist(&dump_path)?;
Ok(dump_path)
@ -341,9 +341,9 @@ mod test {
create_update_handler(index_resolver.clone(), tmp.path(), 4096 * 100).unwrap();
let task = DumpTask {
path: tmp.path().to_owned(),
dump_path: tmp.path().into(),
// this should do nothing
analytics_path: tmp.path().join("user-id"),
db_path: tmp.path().into(),
index_resolver,
update_sender,
uid: String::from("test"),
@ -371,9 +371,9 @@ mod test {
create_update_handler(index_resolver.clone(), tmp.path(), 4096 * 100).unwrap();
let task = DumpTask {
path: tmp.path().to_owned(),
dump_path: tmp.path().into(),
// this should do nothing
analytics_path: tmp.path().join("user-id"),
db_path: tmp.path().into(),
index_resolver,
update_sender,
uid: String::from("test"),

1
meilisearch-lib/src/index_controller/mod.rs
View File

@ -189,6 +189,7 @@ impl IndexControllerBuilder {
.ok_or_else(|| anyhow::anyhow!("Snapshot interval not provided."))?,
self.snapshot_dir
.ok_or_else(|| anyhow::anyhow!("Snapshot path not provided."))?,
db_path.as_ref().into(),
db_path
.as_ref()
.file_name()

12
meilisearch-lib/src/index_controller/snapshot.rs
View File

@ -8,6 +8,7 @@ use tokio::fs;
use tokio::task::spawn_blocking;
use tokio::time::sleep;
use crate::analytics;
use crate::compression::from_tar_gz;
use crate::index_controller::updates::UpdateMsg;
@ -21,6 +22,7 @@ pub struct SnapshotService<U, I> {
update_sender: UpdateSender,
snapshot_period: Duration,
snapshot_path: PathBuf,
db_path: PathBuf,
db_name: String,
}
@ -34,6 +36,7 @@ where
update_sender: UpdateSender,
snapshot_period: Duration,
snapshot_path: PathBuf,
db_path: PathBuf,
db_name: String,
) -> Self {
Self {
@ -41,6 +44,7 @@ where
update_sender,
snapshot_period,
snapshot_path,
db_path,
db_name,
}
}
@ -71,6 +75,8 @@ where
.snapshot(temp_snapshot_path.clone())
.await?;
analytics::copy_user_id(&self.db_path, &temp_snapshot_path.clone());
if indexes.is_empty() {
return Ok(());
}
@ -211,6 +217,8 @@ mod test {
update_sender,
Duration::from_millis(100),
snapshot_path.path().to_owned(),
// this should do nothing
snapshot_path.path().to_owned(),
"data.ms".to_string(),
);
@ -243,6 +251,8 @@ mod test {
update_sender,
Duration::from_millis(100),
snapshot_path.path().to_owned(),
// this should do nothing
snapshot_path.path().to_owned(),
"data.ms".to_string(),
);
@ -292,6 +302,8 @@ mod test {
update_sender,
Duration::from_millis(100),
snapshot_path.path().to_owned(),
// this should do nothing
snapshot_path.path().to_owned(),
"data.ms".to_string(),
);