snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 18:17:39 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

fix all the import and comment most of the dump v6

Browse Source
This commit is contained in:
Tamo 2022-10-11 18:26:49 +02:00 committed by Clément Renault
parent 141a1c9464
commit 0af00f6b32
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
5 changed files with 36 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dump/src/lib.rs
View File

@ -52,10 +52,14 @@ pub(crate) mod test {
}; };
use big_s::S; use big_s::S;
use index::{Checked, Settings}; use index_scheduler::{
use index_scheduler::{milli::update::Setting, task::Details, Kind, Status, TaskView}; task::{Details, DetailsView},
Kind, Status, TaskView,
};
use maplit::btreeset; use maplit::btreeset;
use meilisearch_auth::{Action, Key}; use meilisearch_auth::{Action, Key};
use meilisearch_types::milli::{self, update::Setting};
use meilisearch_types::settings::{Checked, Settings};
use meilisearch_types::{index_uid::IndexUid, star_or::StarOr}; use meilisearch_types::{index_uid::IndexUid, star_or::StarOr};
use serde_json::{json, Map, Value}; use serde_json::{json, Map, Value};
use time::{macros::datetime, Duration}; use time::{macros::datetime, Duration};
@ -122,13 +126,16 @@ pub(crate) mod test {
index_uid: Some(S("doggos")), index_uid: Some(S("doggos")),
status: Status::Succeeded, status: Status::Succeeded,
kind: Kind::DocumentImport { kind: Kind::DocumentImport {
method: index::milli::update::IndexDocumentsMethod::UpdateDocuments, method: milli::update::IndexDocumentsMethod::UpdateDocuments,
allow_index_creation: true, allow_index_creation: true,
}, },
details: Some(Details::DocumentAddition { details: todo!(),
/*
Some(DetailsView::DocumentAddition {
received_documents: 10_000, received_documents: 10_000,
indexed_documents: 3, indexed_documents: 3,
}), }),
*/
error: None, error: None,
duration: Some(Duration::DAY), duration: Some(Duration::DAY),
enqueued_at: datetime!(2022-11-11 0:00 UTC), enqueued_at: datetime!(2022-11-11 0:00 UTC),
@ -143,7 +150,7 @@ pub(crate) mod test {
index_uid: Some(S("doggos")), index_uid: Some(S("doggos")),
status: Status::Enqueued, status: Status::Enqueued,
kind: Kind::DocumentImport { kind: Kind::DocumentImport {
method: index::milli::update::IndexDocumentsMethod::UpdateDocuments, method: milli::update::IndexDocumentsMethod::UpdateDocuments,
allow_index_creation: true, allow_index_creation: true,
}, },
details: None, details: None,

9
dump/src/reader/compat/v5_to_v6.rs
View File

@ -83,10 +83,10 @@ impl CompatV5ToV6 {
} => v6::Kind::DocumentImport { } => v6::Kind::DocumentImport {
method: match merge_strategy { method: match merge_strategy {
v5::tasks::IndexDocumentsMethod::ReplaceDocuments => { v5::tasks::IndexDocumentsMethod::ReplaceDocuments => {
v6::index::milli::update::IndexDocumentsMethod::ReplaceDocuments v6::milli::update::IndexDocumentsMethod::ReplaceDocuments
} }
v5::tasks::IndexDocumentsMethod::UpdateDocuments => { v5::tasks::IndexDocumentsMethod::UpdateDocuments => {
v6::index::milli::update::IndexDocumentsMethod::UpdateDocuments v6::milli::update::IndexDocumentsMethod::UpdateDocuments
} }
}, },
allow_index_creation: allow_index_creation.clone(), allow_index_creation: allow_index_creation.clone(),
@ -102,7 +102,9 @@ impl CompatV5ToV6 {
}, },
v5::tasks::TaskContent::Dump { .. } => v6::Kind::DumpExport, v5::tasks::TaskContent::Dump { .. } => v6::Kind::DumpExport,
}, },
details: task_view.details.map(|details| match details { details: todo!(),
/*
task_view.details.map(|details| match details {
v5::Details::DocumentAddition { v5::Details::DocumentAddition {
received_documents, received_documents,
indexed_documents, indexed_documents,
@ -128,6 +130,7 @@ impl CompatV5ToV6 {
} }
v5::Details::Dump { dump_uid } => v6::Details::Dump { dump_uid }, v5::Details::Dump { dump_uid } => v6::Details::Dump { dump_uid },
}), }),
*/
error: task_view.error.map(|e| e.into()), error: task_view.error.map(|e| e.into()),
duration: task_view.duration, duration: task_view.duration,
enqueued_at: task_view.enqueued_at, enqueued_at: task_view.enqueued_at,

23
dump/src/reader/v6.rs
View File

@ -11,14 +11,15 @@ use uuid::Uuid;
use crate::{Error, IndexMetadata, Result, Version}; use crate::{Error, IndexMetadata, Result, Version};
pub use meilisearch_types::milli;
use super::Document; use super::Document;
pub use index;
pub type Metadata = crate::Metadata; pub type Metadata = crate::Metadata;
pub type Settings<T> = index::Settings<T>; pub type Settings<T> = meilisearch_types::settings::Settings<T>;
pub type Checked = index::Checked; pub type Checked = meilisearch_types::settings::Checked;
pub type Unchecked = index::Unchecked; pub type Unchecked = meilisearch_types::settings::Unchecked;
pub type Task = index_scheduler::TaskView; pub type Task = index_scheduler::TaskView;
pub type Key = meilisearch_auth::Key; pub type Key = meilisearch_auth::Key;
@ -30,11 +31,11 @@ pub type Kind = index_scheduler::Kind;
pub type Details = index_scheduler::Details; pub type Details = index_scheduler::Details;
// everything related to the settings // everything related to the settings
pub type Setting<T> = index::milli::update::Setting<T>; pub type Setting<T> = meilisearch_types::milli::update::Setting<T>;
pub type TypoTolerance = index::updates::TypoSettings; pub type TypoTolerance = meilisearch_types::settings::TypoSettings;
pub type MinWordSizeForTypos = index::updates::MinWordSizeTyposSetting; pub type MinWordSizeForTypos = meilisearch_types::settings::MinWordSizeTyposSetting;
pub type FacetingSettings = index::updates::FacetingSettings; pub type FacetingSettings = meilisearch_types::settings::FacetingSettings;
pub type PaginationSettings = index::updates::PaginationSettings; pub type PaginationSettings = meilisearch_types::settings::PaginationSettings;
// everything related to the api keys // everything related to the api keys
pub type Action = meilisearch_auth::Action; pub type Action = meilisearch_auth::Action;
@ -108,8 +109,8 @@ impl V6Reader {
&mut self, &mut self,
) -> Box<dyn Iterator<Item = Result<(Task, Option<Box<super::UpdateFile>>)>> + '_> { ) -> Box<dyn Iterator<Item = Result<(Task, Option<Box<super::UpdateFile>>)>> + '_> {
Box::new((&mut self.tasks).lines().map(|line| -> Result<_> { Box::new((&mut self.tasks).lines().map(|line| -> Result<_> {
let mut task: index_scheduler::TaskView = serde_json::from_str(&line?)?; let mut task: index_scheduler::TaskView = todo!(); // serde_json::from_str(&line?)?;
// TODO: this can be removed once we can `Deserialize` the duration from the `TaskView`. // TODO: this can be removed once we can `Deserialize` the duration from the `TaskView`.
if let Some((started_at, finished_at)) = task.started_at.zip(task.finished_at) { if let Some((started_at, finished_at)) = task.started_at.zip(task.finished_at) {
task.duration = Some(finished_at - started_at); task.duration = Some(finished_at - started_at);
} }

7
dump/src/writer.rs
View File

@ -5,9 +5,9 @@ use std::{
}; };
use flate2::{write::GzEncoder, Compression}; use flate2::{write::GzEncoder, Compression};
use index::{Checked, Settings};
use index_scheduler::TaskView; use index_scheduler::TaskView;
use meilisearch_auth::Key; use meilisearch_auth::Key;
use meilisearch_types::settings::{Checked, Settings};
use serde_json::{Map, Value}; use serde_json::{Map, Value};
use tempfile::TempDir; use tempfile::TempDir;
use time::OffsetDateTime; use time::OffsetDateTime;
@ -183,7 +183,7 @@ pub(crate) mod test {
use std::{fmt::Write, io::BufReader, path::Path, str::FromStr}; use std::{fmt::Write, io::BufReader, path::Path, str::FromStr};
use flate2::bufread::GzDecoder; use flate2::bufread::GzDecoder;
use index::Unchecked; use meilisearch_types::settings::Unchecked;
use crate::{ use crate::{
reader::Document, reader::Document,
@ -332,7 +332,8 @@ pub(crate) mod test {
for (task, mut expected) in tasks_queue.lines().zip(create_test_tasks()) { for (task, mut expected) in tasks_queue.lines().zip(create_test_tasks()) {
// TODO: This can be removed once `Duration` from the `TaskView` is implemented. // TODO: This can be removed once `Duration` from the `TaskView` is implemented.
expected.0.duration = None; expected.0.duration = None;
assert_eq!(serde_json::from_str::<TaskView>(task).unwrap(), expected.0); // TODO: uncomment this one once the we write the dump integration in the index-scheduler
// assert_eq!(serde_json::from_str::<TaskView>(task).unwrap(), expected.0);
if let Some(expected_update) = expected.1 { if let Some(expected_update) = expected.1 {
let path = dump_path.join(format!("tasks/update_files/{}.jsonl", expected.0.uid)); let path = dump_path.join(format!("tasks/update_files/{}.jsonl", expected.0.uid));

2
index-scheduler/src/task.rs
View File

@ -416,6 +416,7 @@ pub enum Details {
dump_uid: String, dump_uid: String,
}, },
} }
#[derive(Default, Debug, PartialEq, Clone, Serialize)] #[derive(Default, Debug, PartialEq, Clone, Serialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct DetailsView { pub struct DetailsView {
@ -441,6 +442,7 @@ pub struct DetailsView {
#[serde(flatten)] #[serde(flatten)]
settings: Option<Settings<Unchecked>>, settings: Option<Settings<Unchecked>>,
} }
impl Details { impl Details {
fn as_details_view(&self) -> DetailsView { fn as_details_view(&self) -> DetailsView {
match self.clone() { match self.clone() {