Merge #2444

2444: add boilerplate for dump v5 r=MarinPostma a=MarinPostma

add the boilerplate files for dump v5


Co-authored-by: ad hoc <postma.marin@protonmail.com>
Co-authored-by: ManyTheFish <many@meilisearch.com>

meilisearch-lib/src/dump/compat/mod.rs

@@ -1,5 +1,6 @@
 pub mod v2;
 pub mod v3;
+pub mod v4;
 
 /// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name.
 pub fn asc_ranking_rule(text: &str) -> Option<&str> {

meilisearch-lib/src/dump/compat/v4.rs

@@ -0,0 +1 @@
+

meilisearch-lib/src/dump/loaders/mod.rs

@@ -1,3 +1,4 @@
 pub mod v2;
 pub mod v3;
 pub mod v4;
+pub mod v5;

meilisearch-lib/src/dump/loaders/v3.rs

@@ -66,7 +66,6 @@ pub fn load_dump(
         index_db_size,
         meta_env_size,
         indexing_options,
-        "V5",
     )
 }
 

meilisearch-lib/src/dump/loaders/v4.rs

@@ -1,16 +1,12 @@
+use std::fs;
 use std::path::Path;
-use std::sync::Arc;
 
+use fs_extra::dir::{self, CopyOptions};
 use log::info;
-use meilisearch_auth::AuthController;
+use tempfile::tempdir;
-use milli::heed::EnvOpenOptions;
 
-use crate::analytics;
 use crate::dump::Metadata;
-use crate::index_resolver::IndexResolver;
 use crate::options::IndexerOpts;
-use crate::tasks::TaskStore;
-use crate::update_file_store::UpdateFileStore;
 
 pub fn load_dump(
     meta: Metadata,
@@ -19,31 +15,42 @@ pub fn load_dump(
     index_db_size: usize,
     meta_env_size: usize,
     indexing_options: &IndexerOpts,
-    version: &str,
 ) -> anyhow::Result<()> {
-    info!(
+    info!("Patching dump V4 to dump V5...");
-        "Loading dump from {}, dump database version: {}, dump version: {}",
-        meta.dump_date, meta.db_version, version
-    );
 
-    let mut options = EnvOpenOptions::new();
+    let patched_dir = tempdir()?;
-    options.map_size(meta_env_size);
+    let options = CopyOptions::default();
-    options.max_dbs(100);
-    let env = Arc::new(options.open(&dst)?);
 
-    IndexResolver::load_dump(
+    // Indexes
-        src.as_ref(),
+    dir::copy(src.as_ref().join("indexes"), patched_dir.path(), &options)?;
-        &dst,
+
-        index_db_size,
+    // Index uuids
-        env.clone(),
+    dir::copy(
-        indexing_options,
+        src.as_ref().join("index_uuids"),
+        patched_dir.path(),
+        &options,
     )?;
-    UpdateFileStore::load_dump(src.as_ref(), &dst)?;
-    TaskStore::load_dump(&src, env)?;
-    AuthController::load_dump(&src, &dst)?;
-    analytics::copy_user_id(src.as_ref(), dst.as_ref());
 
-    info!("Loading indexes.");
+    // Metadata
+    fs::copy(
+        src.as_ref().join("metadata.json"),
+        patched_dir.path().join("metadata.json"),
+    )?;
 
-    Ok(())
+    // Updates
+    dir::copy(src.as_ref().join("updates"), patched_dir.path(), &options)?;
+
+    // Keys
+    if src.as_ref().join("keys").exists() {
+        fs::copy(src.as_ref().join("keys"), patched_dir.path().join("keys"))?;
+    }
+
+    super::v5::load_dump(
+        meta,
+        patched_dir.path(),
+        dst,
+        index_db_size,
+        meta_env_size,
+        indexing_options,
+    )
 }

meilisearch-lib/src/dump/loaders/v5.rs

@@ -0,0 +1,47 @@
+use std::{path::Path, sync::Arc};
+
+use log::info;
+use meilisearch_auth::AuthController;
+use milli::heed::EnvOpenOptions;
+
+use crate::analytics;
+use crate::dump::Metadata;
+use crate::index_resolver::IndexResolver;
+use crate::options::IndexerOpts;
+use crate::tasks::TaskStore;
+use crate::update_file_store::UpdateFileStore;
+
+pub fn load_dump(
+    meta: Metadata,
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    index_db_size: usize,
+    meta_env_size: usize,
+    indexing_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    info!(
+        "Loading dump from {}, dump database version: {}, dump version: V5",
+        meta.dump_date, meta.db_version
+    );
+
+    let mut options = EnvOpenOptions::new();
+    options.map_size(meta_env_size);
+    options.max_dbs(100);
+    let env = Arc::new(options.open(&dst)?);
+
+    IndexResolver::load_dump(
+        src.as_ref(),
+        &dst,
+        index_db_size,
+        env.clone(),
+        indexing_options,
+    )?;
+    UpdateFileStore::load_dump(src.as_ref(), &dst)?;
+    TaskStore::load_dump(&src, env)?;
+    AuthController::load_dump(&src, &dst)?;
+    analytics::copy_user_id(src.as_ref(), dst.as_ref());
+
+    info!("Loading indexes.");
+
+    Ok(())
+}

meilisearch-lib/src/dump/mod.rs

@@ -11,7 +11,7 @@ use tempfile::TempDir;
 use crate::compression::from_tar_gz;
 use crate::options::IndexerOpts;
 
-use self::loaders::{v2, v3, v4};
+use self::loaders::{v2, v3, v4, v5};
 
 pub use handler::{generate_uid, DumpHandler};
 
@@ -69,7 +69,6 @@ impl MetadataVersion {
         meta_env_size: usize,
         indexing_options: &IndexerOpts,
     ) -> anyhow::Result<()> {
-        let version = self.version();
         match self {
             MetadataVersion::V1(_meta) => {
                 anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.")
@@ -90,14 +89,21 @@ impl MetadataVersion {
                 meta_env_size,
                 indexing_options,
             )?,
-            MetadataVersion::V4(meta) | MetadataVersion::V5(meta) => v4::load_dump(
+            MetadataVersion::V4(meta) => v4::load_dump(
+                meta,
+                src,
+                dst,
+                index_db_size,
+                meta_env_size,
+                indexing_options,
+            )?,
+            MetadataVersion::V5(meta) => v5::load_dump(
                 meta,
                 src,
                 dst,
                 index_db_size,
                 meta_env_size,
                 indexing_options,
-                version,
             )?,
         }