From 446f1f31e01056667a1135e7937f6f3e13cf6f88 Mon Sep 17 00:00:00 2001 From: ad hoc Date: Mon, 30 May 2022 16:42:51 +0200 Subject: [PATCH] rename Succeded to Succeeded --- meilisearch-http/src/routes/tasks.rs | 2 +- meilisearch-http/src/task.rs | 2 +- meilisearch-http/tests/search/mod.rs | 1 - meilisearch-lib/src/dump/compat/v3.rs | 5 +- meilisearch-lib/src/dump/compat/v4.rs | 66 +++++++++++++++++++ meilisearch-lib/src/dump/loaders/v3.rs | 6 +- meilisearch-lib/src/dump/loaders/v4.rs | 31 ++++++++- meilisearch-lib/src/index_resolver/mod.rs | 4 +- .../src/tasks/handlers/dump_handler.rs | 2 +- meilisearch-lib/src/tasks/task.rs | 9 ++- 10 files changed, 111 insertions(+), 17 deletions(-) diff --git a/meilisearch-http/src/routes/tasks.rs b/meilisearch-http/src/routes/tasks.rs index 1fe903abf..5c6ef711e 100644 --- a/meilisearch-http/src/routes/tasks.rs +++ b/meilisearch-http/src/routes/tasks.rs @@ -79,7 +79,7 @@ fn task_status_matches_events(status: &TaskStatus, events: &[TaskEvent]) -> bool matches!((status, event), (TaskStatus::Enqueued, TaskEvent::Created(_)) | (TaskStatus::Processing, TaskEvent::Processing(_) | TaskEvent::Batched { .. }) - | (TaskStatus::Succeeded, TaskEvent::Succeded { .. }) + | (TaskStatus::Succeeded, TaskEvent::Succeeded { .. }) | (TaskStatus::Failed, TaskEvent::Failed { .. }), ) }) diff --git a/meilisearch-http/src/task.rs b/meilisearch-http/src/task.rs index c7aaf0030..56eeabfc8 100644 --- a/meilisearch-http/src/task.rs +++ b/meilisearch-http/src/task.rs @@ -263,7 +263,7 @@ impl From for TaskView { TaskEvent::Created(_) => (TaskStatus::Enqueued, None, None), TaskEvent::Batched { .. } => (TaskStatus::Enqueued, None, None), TaskEvent::Processing(_) => (TaskStatus::Processing, None, None), - TaskEvent::Succeded { timestamp, result } => { + TaskEvent::Succeeded { timestamp, result } => { match (result, &mut details) { ( TaskResult::DocumentAddition { diff --git a/meilisearch-http/tests/search/mod.rs b/meilisearch-http/tests/search/mod.rs index c570e8b06..98893dac5 100644 --- a/meilisearch-http/tests/search/mod.rs +++ b/meilisearch-http/tests/search/mod.rs @@ -496,7 +496,6 @@ async fn search_facet_distribution() { |response, code| { assert_eq!(code, 200, "{}", response); let dist = response["facetDistribution"].as_object().unwrap(); - dbg!(&dist); assert_eq!(dist.len(), 3); assert_eq!( dist["doggos.name"], diff --git a/meilisearch-lib/src/dump/compat/v3.rs b/meilisearch-lib/src/dump/compat/v3.rs index befd70963..164b7153d 100644 --- a/meilisearch-lib/src/dump/compat/v3.rs +++ b/meilisearch-lib/src/dump/compat/v3.rs @@ -4,9 +4,10 @@ use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use uuid::Uuid; +use super::v4::{Task, TaskEvent}; use crate::index::{Settings, Unchecked}; use crate::index_resolver::IndexUid; -use crate::tasks::task::{DocumentDeletion, Task, TaskContent, TaskEvent, TaskId, TaskResult}; +use crate::tasks::task::{DocumentDeletion, TaskContent, TaskId, TaskResult}; use super::v2; @@ -187,7 +188,7 @@ impl From<(UpdateStatus, String, TaskId)> for Task { // Dummy task let mut task = Task { id: task_id, - index_uid: Some(IndexUid::new(uid).unwrap()), + index_uid: IndexUid::new(uid).unwrap(), content: TaskContent::IndexDeletion, events: Vec::new(), }; diff --git a/meilisearch-lib/src/dump/compat/v4.rs b/meilisearch-lib/src/dump/compat/v4.rs index 8b1378917..6fa0e582a 100644 --- a/meilisearch-lib/src/dump/compat/v4.rs +++ b/meilisearch-lib/src/dump/compat/v4.rs @@ -1 +1,67 @@ +use meilisearch_error::ResponseError; +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; +use crate::tasks::batch::BatchId; +use crate::tasks::task::{TaskContent, TaskEvent as NewTaskEvent, TaskId, TaskResult}; +use crate::IndexUid; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Task { + pub id: TaskId, + pub index_uid: IndexUid, + pub content: TaskContent, + pub events: Vec, +} + +impl From for crate::tasks::task::Task { + fn from(other: Task) -> Self { + Self { + id: other.id, + index_uid: Some(other.index_uid), + content: other.content, + events: other.events.into_iter().map(Into::into).collect(), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum TaskEvent { + Created(#[serde(with = "time::serde::rfc3339")] OffsetDateTime), + Batched { + #[serde(with = "time::serde::rfc3339")] + timestamp: OffsetDateTime, + batch_id: BatchId, + }, + Processing(#[serde(with = "time::serde::rfc3339")] OffsetDateTime), + Succeded { + result: TaskResult, + #[serde(with = "time::serde::rfc3339")] + timestamp: OffsetDateTime, + }, + Failed { + error: ResponseError, + #[serde(with = "time::serde::rfc3339")] + timestamp: OffsetDateTime, + }, +} + +impl From for NewTaskEvent { + fn from(other: TaskEvent) -> Self { + match other { + TaskEvent::Created(x) => NewTaskEvent::Created(x), + TaskEvent::Batched { + timestamp, + batch_id, + } => NewTaskEvent::Batched { + timestamp, + batch_id, + }, + TaskEvent::Processing(x) => NewTaskEvent::Processing(x), + TaskEvent::Succeded { result, timestamp } => { + NewTaskEvent::Succeeded { result, timestamp } + } + TaskEvent::Failed { error, timestamp } => NewTaskEvent::Failed { error, timestamp }, + } + } +} diff --git a/meilisearch-lib/src/dump/loaders/v3.rs b/meilisearch-lib/src/dump/loaders/v3.rs index 0a2ea438b..44984c946 100644 --- a/meilisearch-lib/src/dump/loaders/v3.rs +++ b/meilisearch-lib/src/dump/loaders/v3.rs @@ -9,11 +9,11 @@ use log::info; use tempfile::tempdir; use uuid::Uuid; -use crate::dump::compat::v3; +use crate::dump::compat::{self, v3}; use crate::dump::Metadata; use crate::index_resolver::meta_store::{DumpEntry, IndexMeta}; use crate::options::IndexerOpts; -use crate::tasks::task::{Task, TaskId}; +use crate::tasks::task::TaskId; /// dump structure for V3: /// . @@ -124,7 +124,7 @@ fn patch_updates( .clone(); serde_json::to_writer( &mut dst_file, - &Task::from((entry.update, name, task_id as TaskId)), + &compat::v4::Task::from((entry.update, name, task_id as TaskId)), )?; dst_file.write_all(b"\n")?; Ok(()) diff --git a/meilisearch-lib/src/dump/loaders/v4.rs b/meilisearch-lib/src/dump/loaders/v4.rs index 50fffeb9e..126300af8 100644 --- a/meilisearch-lib/src/dump/loaders/v4.rs +++ b/meilisearch-lib/src/dump/loaders/v4.rs @@ -1,12 +1,14 @@ -use std::fs; +use std::fs::{self, create_dir_all, File}; +use std::io::Write; use std::path::Path; use fs_extra::dir::{self, CopyOptions}; use log::info; use tempfile::tempdir; -use crate::dump::Metadata; +use crate::dump::{compat, Metadata}; use crate::options::IndexerOpts; +use crate::tasks::task::Task; pub fn load_dump( meta: Metadata, @@ -38,7 +40,7 @@ pub fn load_dump( )?; // Updates - dir::copy(src.as_ref().join("updates"), patched_dir.path(), &options)?; + patch_updates(&src, &patched_dir)?; // Keys if src.as_ref().join("keys").exists() { @@ -54,3 +56,26 @@ pub fn load_dump( indexing_options, ) } + +fn patch_updates(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { + let updates_path = src.as_ref().join("updates/data.jsonl"); + let output_updates_path = dst.as_ref().join("updates/data.jsonl"); + create_dir_all(output_updates_path.parent().unwrap())?; + let udpates_file = File::open(updates_path)?; + let mut output_update_file = File::create(output_updates_path)?; + + serde_json::Deserializer::from_reader(udpates_file) + .into_iter::() + .try_for_each(|task| -> anyhow::Result<()> { + let task: Task = task?.into(); + + serde_json::to_writer(&mut output_update_file, &task)?; + output_update_file.write_all(b"\n")?; + + Ok(()) + })?; + + output_update_file.flush()?; + + Ok(()) +} diff --git a/meilisearch-lib/src/index_resolver/mod.rs b/meilisearch-lib/src/index_resolver/mod.rs index 1900061c7..33b480f61 100644 --- a/meilisearch-lib/src/index_resolver/mod.rs +++ b/meilisearch-lib/src/index_resolver/mod.rs @@ -200,7 +200,7 @@ where .await; let event = match result { - Ok(Ok(result)) => TaskEvent::Succeded { + Ok(Ok(result)) => TaskEvent::Succeeded { timestamp: OffsetDateTime::now_utc(), result: TaskResult::DocumentAddition { indexed_documents: result.indexed_documents, @@ -594,7 +594,7 @@ mod test { { assert!(matches!(result.content.first().unwrap().events.last().unwrap(), TaskEvent::Failed { .. }), "{:?}", result); } else { - assert!(matches!(result.content.first().unwrap().events.last().unwrap(), TaskEvent::Succeded { .. }), "{:?}", result); + assert!(matches!(result.content.first().unwrap().events.last().unwrap(), TaskEvent::Succeeded { .. }), "{:?}", result); } } }); diff --git a/meilisearch-lib/src/tasks/handlers/dump_handler.rs b/meilisearch-lib/src/tasks/handlers/dump_handler.rs index 715beafee..c708dadcc 100644 --- a/meilisearch-lib/src/tasks/handlers/dump_handler.rs +++ b/meilisearch-lib/src/tasks/handlers/dump_handler.rs @@ -91,7 +91,7 @@ mod test { if accept { let batch = dump_handler.process_batch(batch).await; let last_event = batch.content.first().unwrap().events.last().unwrap(); - assert!(matches!(last_event, TaskEvent::Succeded { .. })); + assert!(matches!(last_event, TaskEvent::Succeeded { .. })); } }); diff --git a/meilisearch-lib/src/tasks/task.rs b/meilisearch-lib/src/tasks/task.rs index 0499d9702..97eb11467 100644 --- a/meilisearch-lib/src/tasks/task.rs +++ b/meilisearch-lib/src/tasks/task.rs @@ -48,7 +48,7 @@ pub enum TaskEvent { #[serde(with = "time::serde::rfc3339")] OffsetDateTime, ), - Succeded { + Succeeded { result: TaskResult, #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] #[serde(with = "time::serde::rfc3339")] @@ -64,7 +64,7 @@ pub enum TaskEvent { impl TaskEvent { pub fn succeeded(result: TaskResult) -> Self { - Self::Succeded { + Self::Succeeded { result, timestamp: OffsetDateTime::now_utc(), } @@ -106,7 +106,10 @@ impl Task { /// A task is finished when its last state is either `Succeeded` or `Failed`. pub fn is_finished(&self) -> bool { self.events.last().map_or(false, |event| { - matches!(event, TaskEvent::Succeded { .. } | TaskEvent::Failed { .. }) + matches!( + event, + TaskEvent::Succeeded { .. } | TaskEvent::Failed { .. } + ) }) }