diff --git a/config.toml b/config.toml index 8933dd22f..6ef9b77f1 100644 --- a/config.toml +++ b/config.toml @@ -56,7 +56,7 @@ disable_auto_batching = false ### DUMPS ### ############# -dumps_dir = "dumps/" +dump_dir = "dumps/" # Sets the directory where Meilisearch will create dump files. # https://docs.meilisearch.com/learn/configuration/instance_options.html#dumps-destination diff --git a/dump/src/reader/compat/v1_to_v2.rs b/dump/src/reader/compat/v1_to_v2.rs index 23e4529dc..741d18fa8 100644 --- a/dump/src/reader/compat/v1_to_v2.rs +++ b/dump/src/reader/compat/v1_to_v2.rs @@ -1,8 +1,8 @@ -use std::{collections::BTreeSet, str::FromStr}; - -use crate::reader::{v1, v2, Document}; +use std::collections::BTreeSet; +use std::str::FromStr; use super::v2_to_v3::CompatV2ToV3; +use crate::reader::{v1, v2, Document}; use crate::Result; pub struct CompatV1ToV2 { @@ -367,12 +367,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:23:39.976870431Z", + "updatedAt": "2022-10-02T13:27:54.353262482Z" } "###); @@ -382,12 +382,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:15:29.477512777Z", + "updatedAt": "2022-10-02T13:21:12.671204856Z" } "###); @@ -397,12 +397,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:38:26.358882984Z", + "updatedAt": "2022-10-02T13:38:26.385609433Z" } "###); diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index efbca06d0..6414c55d4 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -299,12 +299,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:39.360187055Z", + "updatedAt": "2022-10-06T12:53:40.603035979Z" } "###); @@ -314,12 +314,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:38.710611568Z", + "updatedAt": "2022-10-06T12:53:49.785862546Z" } "###); @@ -329,12 +329,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:40.831649057Z", + "updatedAt": "2022-10-06T12:53:41.116036186Z" } "###); @@ -562,12 +562,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:23:39.976870431Z", + "updatedAt": "2022-10-02T13:27:54.353262482Z" } "###); @@ -577,12 +577,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:15:29.477512777Z", + "updatedAt": "2022-10-02T13:21:12.671204856Z" } "###); @@ -592,12 +592,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:38:26.358882984Z", + "updatedAt": "2022-10-02T13:38:26.385609433Z" } "###); diff --git a/dump/src/reader/v1/mod.rs b/dump/src/reader/v1/mod.rs index 1932b602a..ac7324d9a 100644 --- a/dump/src/reader/v1/mod.rs +++ b/dump/src/reader/v1/mod.rs @@ -1,15 +1,14 @@ -use std::{ - fs::{self, File}, - io::{BufRead, BufReader}, - path::{Path, PathBuf}, -}; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use serde::Deserialize; use tempfile::TempDir; use time::OffsetDateTime; -use super::{compat::v1_to_v2::CompatV1ToV2, Document}; +use super::compat::v1_to_v2::CompatV1ToV2; +use super::Document; use crate::{IndexMetadata, Result, Version}; -use serde::Deserialize; pub mod settings; pub mod update; @@ -204,12 +203,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:23:39.976870431Z", + "updatedAt": "2022-10-02T13:27:54.353262482Z" } "###); @@ -223,12 +222,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"91de507f206ad21964584021932ba7a7"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:15:29.477512777Z", + "updatedAt": "2022-10-02T13:21:12.671204856Z" } "###); @@ -242,12 +241,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"55eef4de2bef7e84c5ce0bee47488f56"); // spells - insta::assert_json_snapshot!(dnd_spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(dnd_spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-02T13:38:26.358882984Z", + "updatedAt": "2022-10-02T13:38:26.385609433Z" } "###); diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 30fc27657..4dfbf3502 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -13,7 +13,7 @@ pub mod meta; pub mod settings; pub mod tasks; -use self::meta::{DumpMeta, IndexUuid}; +use self::meta::{DumpMeta, IndexMeta, IndexUuid}; use super::compat::v4_to_v5::CompatV4ToV5; use crate::{Error, IndexMetadata, Result, Version}; @@ -100,6 +100,10 @@ impl V4Reader { V4IndexReader::new( index.uid.clone(), &self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()), + &index.index_meta, + BufReader::new( + File::open(&self.dump.path().join("updates").join("data.jsonl")).unwrap(), + ), ) })) } @@ -147,16 +151,44 @@ pub struct V4IndexReader { } impl V4IndexReader { - pub fn new(name: String, path: &Path) -> Result { + pub fn new( + name: String, + path: &Path, + index_metadata: &IndexMeta, + tasks: BufReader, + ) -> Result { let meta = File::open(path.join("meta.json"))?; let meta: DumpMeta = serde_json::from_reader(meta)?; + let mut created_at = None; + let mut updated_at = None; + + for line in tasks.lines() { + let task: Task = serde_json::from_str(&line?)?; + + if task.index_uid.to_string() == name { + // The first task to match our index_uid that succeeded (ie. processed_at returns Some) + // is our `last_updated_at`. + if updated_at.is_none() { + updated_at = task.processed_at() + } + + // Once we reach the `creation_task_id` we can stop iterating on the task queue and + // this task represents our `created_at`. + if task.id as usize == index_metadata.creation_task_id { + created_at = task.created_at(); + break; + } + } + } + + let current_time = OffsetDateTime::now_utc(); + let metadata = IndexMetadata { uid: name, primary_key: meta.primary_key, - // FIXME: Iterate over the whole task queue to find the creation and last update date. - created_at: OffsetDateTime::now_utc(), - updated_at: OffsetDateTime::now_utc(), + created_at: created_at.unwrap_or(current_time), + updated_at: updated_at.unwrap_or(current_time), }; let ret = V4IndexReader { @@ -259,12 +291,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:39.360187055Z", + "updatedAt": "2022-10-06T12:53:40.603035979Z" } "###); @@ -274,12 +306,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:38.710611568Z", + "updatedAt": "2022-10-06T12:53:49.785862546Z" } "###); @@ -289,12 +321,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:40.831649057Z", + "updatedAt": "2022-10-06T12:53:41.116036186Z" } "###); diff --git a/dump/src/reader/v4/tasks.rs b/dump/src/reader/v4/tasks.rs index e1bdde0c7..a701d837d 100644 --- a/dump/src/reader/v4/tasks.rs +++ b/dump/src/reader/v4/tasks.rs @@ -104,6 +104,41 @@ impl Task { }) } + pub fn processed_at(&self) -> Option { + match self.events.last() { + Some(TaskEvent::Succeded { result: _, timestamp }) => Some(*timestamp), + _ => None, + } + } + + pub fn created_at(&self) -> Option { + match &self.content { + TaskContent::IndexCreation { primary_key: _ } => match self.events.first() { + Some(TaskEvent::Created(ts)) => Some(*ts), + _ => None, + }, + TaskContent::DocumentAddition { + content_uuid: _, + merge_strategy: _, + primary_key: _, + documents_count: _, + allow_index_creation: _, + } => match self.events.first() { + Some(TaskEvent::Created(ts)) => Some(*ts), + _ => None, + }, + TaskContent::SettingsUpdate { + settings: _, + is_deletion: _, + allow_index_creation: _, + } => match self.events.first() { + Some(TaskEvent::Created(ts)) => Some(*ts), + _ => None, + }, + _ => None, + } + } + /// Return the content_uuid of the `Task` if there is one. pub fn get_content_uuid(&self) -> Option { match self { diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 6a4cc0336..1d451f9d0 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -119,5 +119,5 @@ japanese = ["meilisearch-types/japanese"] thai = ["meilisearch-types/thai"] [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.3/build.zip" -sha1 = "fb893012023cc33090c549e0eaf10adff335cf6f" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.4/build.zip" +sha1 = "b53c2edb51d4ce1984d5586333b91c4ad3a1b4e4" diff --git a/meilisearch-http/src/analytics/segment_analytics.rs b/meilisearch-http/src/analytics/segment_analytics.rs index afec4c5cb..a8fe9fead 100644 --- a/meilisearch-http/src/analytics/segment_analytics.rs +++ b/meilisearch-http/src/analytics/segment_analytics.rs @@ -216,7 +216,7 @@ struct Infos { env: String, db_path: bool, import_dump: bool, - dumps_dir: bool, + dump_dir: bool, ignore_missing_dump: bool, ignore_dump_if_db_exists: bool, import_snapshot: bool, @@ -272,7 +272,7 @@ impl From for Infos { import_dump, ignore_missing_dump, ignore_dump_if_db_exists, - dumps_dir, + dump_dir, log_level, indexer_options, scheduler_options, @@ -295,7 +295,7 @@ impl From for Infos { env, db_path: db_path != PathBuf::from("./data.ms"), import_dump: import_dump.is_some(), - dumps_dir: dumps_dir != PathBuf::from("dumps/"), + dump_dir: dump_dir != PathBuf::from("dumps/"), ignore_missing_dump, ignore_dump_if_db_exists, import_snapshot: import_snapshot.is_some(), diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 6fa6b77d8..b11f063d2 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -121,7 +121,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Auth update_file_path: opt.db_path.join("update_files"), indexes_path: opt.db_path.join("indexes"), snapshots_path: opt.snapshot_dir.clone(), - dumps_path: opt.dumps_dir.clone(), + dumps_path: opt.dump_dir.clone(), task_db_size: opt.max_task_db_size.get_bytes() as usize, index_size: opt.max_index_size.get_bytes() as usize, indexer_config: (&opt.indexer_options).try_into()?, diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index 82d67d5a0..0d4ce133a 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -47,7 +47,7 @@ const MEILI_SNAPSHOT_INTERVAL_SEC: &str = "MEILI_SNAPSHOT_INTERVAL_SEC"; const MEILI_IMPORT_DUMP: &str = "MEILI_IMPORT_DUMP"; const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP"; const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS"; -const MEILI_DUMPS_DIR: &str = "MEILI_DUMPS_DIR"; +const MEILI_DUMP_DIR: &str = "MEILI_DUMP_DIR"; const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL"; #[cfg(feature = "metrics")] const MEILI_ENABLE_METRICS_ROUTE: &str = "MEILI_ENABLE_METRICS_ROUTE"; @@ -61,7 +61,7 @@ const DEFAULT_MAX_TASK_DB_SIZE: &str = "100 GiB"; const DEFAULT_HTTP_PAYLOAD_SIZE_LIMIT: &str = "100 MB"; const DEFAULT_SNAPSHOT_DIR: &str = "snapshots/"; const DEFAULT_SNAPSHOT_INTERVAL_SEC: u64 = 86400; -const DEFAULT_DUMPS_DIR: &str = "dumps/"; +const DEFAULT_DUMP_DIR: &str = "dumps/"; const DEFAULT_LOG_LEVEL: &str = "INFO"; const MEILI_MAX_INDEXING_MEMORY: &str = "MEILI_MAX_INDEXING_MEMORY"; @@ -219,9 +219,9 @@ pub struct Opt { pub ignore_dump_if_db_exists: bool, /// Sets the directory where Meilisearch will create dump files. - #[clap(long, env = MEILI_DUMPS_DIR, default_value_os_t = default_dumps_dir())] - #[serde(default = "default_dumps_dir")] - pub dumps_dir: PathBuf, + #[clap(long, env = MEILI_DUMP_DIR, default_value_os_t = default_dump_dir())] + #[serde(default = "default_dump_dir")] + pub dump_dir: PathBuf, /// Defines how much detail should be present in Meilisearch's logs. /// @@ -320,7 +320,7 @@ impl Opt { snapshot_dir, schedule_snapshot, snapshot_interval_sec, - dumps_dir, + dump_dir, log_level, indexer_options, scheduler_options, @@ -373,7 +373,7 @@ impl Opt { MEILI_SNAPSHOT_INTERVAL_SEC, snapshot_interval_sec.to_string(), ); - export_to_env_if_not_present(MEILI_DUMPS_DIR, dumps_dir); + export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir); export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level); #[cfg(feature = "metrics")] { @@ -708,8 +708,8 @@ fn default_snapshot_interval_sec() -> u64 { DEFAULT_SNAPSHOT_INTERVAL_SEC } -fn default_dumps_dir() -> PathBuf { - PathBuf::from(DEFAULT_DUMPS_DIR) +fn default_dump_dir() -> PathBuf { + PathBuf::from(DEFAULT_DUMP_DIR) } fn default_log_level() -> String { diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 3f72248c5..f96c659a4 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -184,7 +184,7 @@ impl Server { pub fn default_settings(dir: impl AsRef) -> Opt { Opt { db_path: dir.as_ref().join("db"), - dumps_dir: dir.as_ref().join("dump"), + dump_dir: dir.as_ref().join("dumps"), env: "development".to_owned(), #[cfg(all(not(debug_assertions), feature = "analytics"))] no_analytics: true,