From 0af00f6b329f48a721f082c7ce7f3b5327e4394d Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 11 Oct 2022 18:26:49 +0200 Subject: [PATCH] fix all the import and comment most of the dump v6 --- dump/src/lib.rs | 17 ++++++++++++----- dump/src/reader/compat/v5_to_v6.rs | 9 ++++++--- dump/src/reader/v6.rs | 23 ++++++++++++----------- dump/src/writer.rs | 7 ++++--- index-scheduler/src/task.rs | 2 ++ 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/dump/src/lib.rs b/dump/src/lib.rs index f2090dace..1b3a4c90d 100644 --- a/dump/src/lib.rs +++ b/dump/src/lib.rs @@ -52,10 +52,14 @@ pub(crate) mod test { }; use big_s::S; - use index::{Checked, Settings}; - use index_scheduler::{milli::update::Setting, task::Details, Kind, Status, TaskView}; + use index_scheduler::{ + task::{Details, DetailsView}, + Kind, Status, TaskView, + }; use maplit::btreeset; use meilisearch_auth::{Action, Key}; + use meilisearch_types::milli::{self, update::Setting}; + use meilisearch_types::settings::{Checked, Settings}; use meilisearch_types::{index_uid::IndexUid, star_or::StarOr}; use serde_json::{json, Map, Value}; use time::{macros::datetime, Duration}; @@ -122,13 +126,16 @@ pub(crate) mod test { index_uid: Some(S("doggos")), status: Status::Succeeded, kind: Kind::DocumentImport { - method: index::milli::update::IndexDocumentsMethod::UpdateDocuments, + method: milli::update::IndexDocumentsMethod::UpdateDocuments, allow_index_creation: true, }, - details: Some(Details::DocumentAddition { + details: todo!(), + /* + Some(DetailsView::DocumentAddition { received_documents: 10_000, indexed_documents: 3, }), + */ error: None, duration: Some(Duration::DAY), enqueued_at: datetime!(2022-11-11 0:00 UTC), @@ -143,7 +150,7 @@ pub(crate) mod test { index_uid: Some(S("doggos")), status: Status::Enqueued, kind: Kind::DocumentImport { - method: index::milli::update::IndexDocumentsMethod::UpdateDocuments, + method: milli::update::IndexDocumentsMethod::UpdateDocuments, allow_index_creation: true, }, details: None, diff --git a/dump/src/reader/compat/v5_to_v6.rs b/dump/src/reader/compat/v5_to_v6.rs index eb4d2d0e7..062524f43 100644 --- a/dump/src/reader/compat/v5_to_v6.rs +++ b/dump/src/reader/compat/v5_to_v6.rs @@ -83,10 +83,10 @@ impl CompatV5ToV6 { } => v6::Kind::DocumentImport { method: match merge_strategy { v5::tasks::IndexDocumentsMethod::ReplaceDocuments => { - v6::index::milli::update::IndexDocumentsMethod::ReplaceDocuments + v6::milli::update::IndexDocumentsMethod::ReplaceDocuments } v5::tasks::IndexDocumentsMethod::UpdateDocuments => { - v6::index::milli::update::IndexDocumentsMethod::UpdateDocuments + v6::milli::update::IndexDocumentsMethod::UpdateDocuments } }, allow_index_creation: allow_index_creation.clone(), @@ -102,7 +102,9 @@ impl CompatV5ToV6 { }, v5::tasks::TaskContent::Dump { .. } => v6::Kind::DumpExport, }, - details: task_view.details.map(|details| match details { + details: todo!(), + /* + task_view.details.map(|details| match details { v5::Details::DocumentAddition { received_documents, indexed_documents, @@ -128,6 +130,7 @@ impl CompatV5ToV6 { } v5::Details::Dump { dump_uid } => v6::Details::Dump { dump_uid }, }), + */ error: task_view.error.map(|e| e.into()), duration: task_view.duration, enqueued_at: task_view.enqueued_at, diff --git a/dump/src/reader/v6.rs b/dump/src/reader/v6.rs index 685b1d3eb..8a6b1ff53 100644 --- a/dump/src/reader/v6.rs +++ b/dump/src/reader/v6.rs @@ -11,14 +11,15 @@ use uuid::Uuid; use crate::{Error, IndexMetadata, Result, Version}; +pub use meilisearch_types::milli; + use super::Document; -pub use index; pub type Metadata = crate::Metadata; -pub type Settings = index::Settings; -pub type Checked = index::Checked; -pub type Unchecked = index::Unchecked; +pub type Settings = meilisearch_types::settings::Settings; +pub type Checked = meilisearch_types::settings::Checked; +pub type Unchecked = meilisearch_types::settings::Unchecked; pub type Task = index_scheduler::TaskView; pub type Key = meilisearch_auth::Key; @@ -30,11 +31,11 @@ pub type Kind = index_scheduler::Kind; pub type Details = index_scheduler::Details; // everything related to the settings -pub type Setting = index::milli::update::Setting; -pub type TypoTolerance = index::updates::TypoSettings; -pub type MinWordSizeForTypos = index::updates::MinWordSizeTyposSetting; -pub type FacetingSettings = index::updates::FacetingSettings; -pub type PaginationSettings = index::updates::PaginationSettings; +pub type Setting = meilisearch_types::milli::update::Setting; +pub type TypoTolerance = meilisearch_types::settings::TypoSettings; +pub type MinWordSizeForTypos = meilisearch_types::settings::MinWordSizeTyposSetting; +pub type FacetingSettings = meilisearch_types::settings::FacetingSettings; +pub type PaginationSettings = meilisearch_types::settings::PaginationSettings; // everything related to the api keys pub type Action = meilisearch_auth::Action; @@ -108,8 +109,8 @@ impl V6Reader { &mut self, ) -> Box>)>> + '_> { Box::new((&mut self.tasks).lines().map(|line| -> Result<_> { - let mut task: index_scheduler::TaskView = serde_json::from_str(&line?)?; - // TODO: this can be removed once we can `Deserialize` the duration from the `TaskView`. + let mut task: index_scheduler::TaskView = todo!(); // serde_json::from_str(&line?)?; + // TODO: this can be removed once we can `Deserialize` the duration from the `TaskView`. if let Some((started_at, finished_at)) = task.started_at.zip(task.finished_at) { task.duration = Some(finished_at - started_at); } diff --git a/dump/src/writer.rs b/dump/src/writer.rs index ba2caa85f..f7b68b5ae 100644 --- a/dump/src/writer.rs +++ b/dump/src/writer.rs @@ -5,9 +5,9 @@ use std::{ }; use flate2::{write::GzEncoder, Compression}; -use index::{Checked, Settings}; use index_scheduler::TaskView; use meilisearch_auth::Key; +use meilisearch_types::settings::{Checked, Settings}; use serde_json::{Map, Value}; use tempfile::TempDir; use time::OffsetDateTime; @@ -183,7 +183,7 @@ pub(crate) mod test { use std::{fmt::Write, io::BufReader, path::Path, str::FromStr}; use flate2::bufread::GzDecoder; - use index::Unchecked; + use meilisearch_types::settings::Unchecked; use crate::{ reader::Document, @@ -332,7 +332,8 @@ pub(crate) mod test { for (task, mut expected) in tasks_queue.lines().zip(create_test_tasks()) { // TODO: This can be removed once `Duration` from the `TaskView` is implemented. expected.0.duration = None; - assert_eq!(serde_json::from_str::(task).unwrap(), expected.0); + // TODO: uncomment this one once the we write the dump integration in the index-scheduler + // assert_eq!(serde_json::from_str::(task).unwrap(), expected.0); if let Some(expected_update) = expected.1 { let path = dump_path.join(format!("tasks/update_files/{}.jsonl", expected.0.uid)); diff --git a/index-scheduler/src/task.rs b/index-scheduler/src/task.rs index aecb0b1b5..1b2e167c8 100644 --- a/index-scheduler/src/task.rs +++ b/index-scheduler/src/task.rs @@ -416,6 +416,7 @@ pub enum Details { dump_uid: String, }, } + #[derive(Default, Debug, PartialEq, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct DetailsView { @@ -441,6 +442,7 @@ pub struct DetailsView { #[serde(flatten)] settings: Option>, } + impl Details { fn as_details_view(&self) -> DetailsView { match self.clone() {