From b0cf431614a0d98254a1c0adce6218b6f6d58ad1 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 5 Dec 2022 18:08:35 +0100 Subject: [PATCH 1/4] Fix the dump date-import of the dumpv4 --- dump/src/reader/v4/mod.rs | 31 +++++++++++++++---------------- dump/src/reader/v4/tasks.rs | 7 ------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 38a81fd05..104896353 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -167,17 +167,16 @@ impl V4IndexReader { let task: Task = serde_json::from_str(&line?)?; if task.index_uid.to_string() == name { + // The first task to match our index_uid that succeeded (ie. processed_at returns Some) + // is our `last_updated_at`. if updated_at.is_none() { - updated_at = task.updated_at() - } - - if created_at.is_none() { - created_at = task.created_at() + updated_at = task.processed_at() } + // Once we reach the `creation_task_id` we can stop iterating on the task queue and + // this task represent our `created_at`. if task.id as usize == index_metadata.creation_task_id { - created_at = task.processed_at(); - + created_at = task.created_at(); break; } } @@ -290,12 +289,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:39.360187055Z", + "updatedAt": "2022-10-06T12:53:40.603035979Z" } "###); @@ -305,12 +304,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:38.710611568Z", + "updatedAt": "2022-10-06T12:53:49.785862546Z" } "###); @@ -320,12 +319,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:40.831649057Z", + "updatedAt": "2022-10-06T12:53:41.116036186Z" } "###); diff --git a/dump/src/reader/v4/tasks.rs b/dump/src/reader/v4/tasks.rs index c9920ffd6..a701d837d 100644 --- a/dump/src/reader/v4/tasks.rs +++ b/dump/src/reader/v4/tasks.rs @@ -111,13 +111,6 @@ impl Task { } } - pub fn updated_at(&self) -> Option { - match self.events.last() { - Some(TaskEvent::Created(ts)) => Some(*ts), - _ => None, - } - } - pub fn created_at(&self) -> Option { match &self.content { TaskContent::IndexCreation { primary_key: _ } => match self.events.first() { From 9c89e3dadccf366c3c72e0fce7c16edc19f4a768 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 5 Dec 2022 18:15:29 +0100 Subject: [PATCH 2/4] uncomment more test for the dump v4 --- dump/src/reader/mod.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index efbca06d0..2b8d77991 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -299,12 +299,12 @@ pub(crate) mod test { assert!(indexes.is_empty()); // products - insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(products.metadata(), @r###" { "uid": "products", "primaryKey": "sku", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:39.360187055Z", + "updatedAt": "2022-10-06T12:53:40.603035979Z" } "###); @@ -314,12 +314,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); // movies - insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(movies.metadata(), @r###" { "uid": "movies", "primaryKey": "id", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:38.710611568Z", + "updatedAt": "2022-10-06T12:53:49.785862546Z" } "###); @@ -329,12 +329,12 @@ pub(crate) mod test { meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab"); // spells - insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + insta::assert_json_snapshot!(spells.metadata(), @r###" { "uid": "dnd_spells", "primaryKey": "index", - "createdAt": "[now]", - "updatedAt": "[now]" + "createdAt": "2022-10-06T12:53:40.831649057Z", + "updatedAt": "2022-10-06T12:53:41.116036186Z" } "###); From 180511795b3603f6a169bfa36c7530c069393503 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 6 Dec 2022 10:53:43 +0100 Subject: [PATCH 3/4] Update dump/src/reader/v4/mod.rs fix typo Co-authored-by: Louis Dureuil --- dump/src/reader/v4/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 104896353..6f10477ad 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -174,7 +174,7 @@ impl V4IndexReader { } // Once we reach the `creation_task_id` we can stop iterating on the task queue and - // this task represent our `created_at`. + // this task represents our `created_at`. if task.id as usize == index_metadata.creation_task_id { created_at = task.created_at(); break; From bef81065f9c2fbd542ccf6579b4a7a830c1adc5a Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 6 Dec 2022 11:03:23 +0100 Subject: [PATCH 4/4] return the same time in case we didn't found a created or updated at --- dump/src/reader/v4/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dump/src/reader/v4/mod.rs b/dump/src/reader/v4/mod.rs index 6f10477ad..4dfbf3502 100644 --- a/dump/src/reader/v4/mod.rs +++ b/dump/src/reader/v4/mod.rs @@ -182,11 +182,13 @@ impl V4IndexReader { } } + let current_time = OffsetDateTime::now_utc(); + let metadata = IndexMetadata { uid: name, primary_key: meta.primary_key, - created_at: created_at.unwrap_or_else(OffsetDateTime::now_utc), - updated_at: updated_at.unwrap_or_else(OffsetDateTime::now_utc), + created_at: created_at.unwrap_or(current_time), + updated_at: updated_at.unwrap_or(current_time), }; let ret = V4IndexReader {