snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 18:17:39 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

fix all the import and comment most of the dump v6

Browse Source
This commit is contained in:
Tamo 2022-10-11 18:26:49 +02:00 committed by Clément Renault
parent 141a1c9464
commit 0af00f6b32
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
5 changed files with 36 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dump/src/lib.rs
View File

@ -52,10 +52,14 @@ pub(crate) mod test {
};
use big_s::S;
use index::{Checked, Settings};
use index_scheduler::{milli::update::Setting, task::Details, Kind, Status, TaskView};
use index_scheduler::{
task::{Details, DetailsView},
Kind, Status, TaskView,
};
use maplit::btreeset;
use meilisearch_auth::{Action, Key};
use meilisearch_types::milli::{self, update::Setting};
use meilisearch_types::settings::{Checked, Settings};
use meilisearch_types::{index_uid::IndexUid, star_or::StarOr};
use serde_json::{json, Map, Value};
use time::{macros::datetime, Duration};
@ -122,13 +126,16 @@ pub(crate) mod test {
index_uid: Some(S("doggos")),
status: Status::Succeeded,
kind: Kind::DocumentImport {
method: index::milli::update::IndexDocumentsMethod::UpdateDocuments,
method: milli::update::IndexDocumentsMethod::UpdateDocuments,
allow_index_creation: true,
},
details: Some(Details::DocumentAddition {
details: todo!(),
/*
Some(DetailsView::DocumentAddition {
received_documents: 10_000,
indexed_documents: 3,
}),
*/
error: None,
duration: Some(Duration::DAY),
enqueued_at: datetime!(2022-11-11 0:00 UTC),
@ -143,7 +150,7 @@ pub(crate) mod test {
index_uid: Some(S("doggos")),
status: Status::Enqueued,
kind: Kind::DocumentImport {
method: index::milli::update::IndexDocumentsMethod::UpdateDocuments,
method: milli::update::IndexDocumentsMethod::UpdateDocuments,
allow_index_creation: true,
},
details: None,

9
dump/src/reader/compat/v5_to_v6.rs
View File

@ -83,10 +83,10 @@ impl CompatV5ToV6 {
} => v6::Kind::DocumentImport {
method: match merge_strategy {
v5::tasks::IndexDocumentsMethod::ReplaceDocuments => {
v6::index::milli::update::IndexDocumentsMethod::ReplaceDocuments
v6::milli::update::IndexDocumentsMethod::ReplaceDocuments
}
v5::tasks::IndexDocumentsMethod::UpdateDocuments => {
v6::index::milli::update::IndexDocumentsMethod::UpdateDocuments
v6::milli::update::IndexDocumentsMethod::UpdateDocuments
}
},
allow_index_creation: allow_index_creation.clone(),
@ -102,7 +102,9 @@ impl CompatV5ToV6 {
},
v5::tasks::TaskContent::Dump { .. } => v6::Kind::DumpExport,
},
details: task_view.details.map(|details| match details {
details: todo!(),
/*
task_view.details.map(|details| match details {
v5::Details::DocumentAddition {
received_documents,
indexed_documents,
@ -128,6 +130,7 @@ impl CompatV5ToV6 {
}
v5::Details::Dump { dump_uid } => v6::Details::Dump { dump_uid },
}),
*/
error: task_view.error.map(|e| e.into()),
duration: task_view.duration,
enqueued_at: task_view.enqueued_at,

21
dump/src/reader/v6.rs
View File

@ -11,14 +11,15 @@ use uuid::Uuid;
use crate::{Error, IndexMetadata, Result, Version};
pub use meilisearch_types::milli;
use super::Document;
pub use index;
pub type Metadata = crate::Metadata;
pub type Settings<T> = index::Settings<T>;
pub type Checked = index::Checked;
pub type Unchecked = index::Unchecked;
pub type Settings<T> = meilisearch_types::settings::Settings<T>;
pub type Checked = meilisearch_types::settings::Checked;
pub type Unchecked = meilisearch_types::settings::Unchecked;
pub type Task = index_scheduler::TaskView;
pub type Key = meilisearch_auth::Key;
@ -30,11 +31,11 @@ pub type Kind = index_scheduler::Kind;
pub type Details = index_scheduler::Details;
// everything related to the settings
pub type Setting<T> = index::milli::update::Setting<T>;
pub type TypoTolerance = index::updates::TypoSettings;
pub type MinWordSizeForTypos = index::updates::MinWordSizeTyposSetting;
pub type FacetingSettings = index::updates::FacetingSettings;
pub type PaginationSettings = index::updates::PaginationSettings;
pub type Setting<T> = meilisearch_types::milli::update::Setting<T>;
pub type TypoTolerance = meilisearch_types::settings::TypoSettings;
pub type MinWordSizeForTypos = meilisearch_types::settings::MinWordSizeTyposSetting;
pub type FacetingSettings = meilisearch_types::settings::FacetingSettings;
pub type PaginationSettings = meilisearch_types::settings::PaginationSettings;
// everything related to the api keys
pub type Action = meilisearch_auth::Action;
@ -108,7 +109,7 @@ impl V6Reader {
&mut self,
) -> Box<dyn Iterator<Item = Result<(Task, Option<Box<super::UpdateFile>>)>> + '_> {
Box::new((&mut self.tasks).lines().map(|line| -> Result<_> {
let mut task: index_scheduler::TaskView = serde_json::from_str(&line?)?;
let mut task: index_scheduler::TaskView = todo!(); // serde_json::from_str(&line?)?;
// TODO: this can be removed once we can `Deserialize` the duration from the `TaskView`.
if let Some((started_at, finished_at)) = task.started_at.zip(task.finished_at) {
task.duration = Some(finished_at - started_at);

7
dump/src/writer.rs
View File

@ -5,9 +5,9 @@ use std::{
};
use flate2::{write::GzEncoder, Compression};
use index::{Checked, Settings};
use index_scheduler::TaskView;
use meilisearch_auth::Key;
use meilisearch_types::settings::{Checked, Settings};
use serde_json::{Map, Value};
use tempfile::TempDir;
use time::OffsetDateTime;
@ -183,7 +183,7 @@ pub(crate) mod test {
use std::{fmt::Write, io::BufReader, path::Path, str::FromStr};
use flate2::bufread::GzDecoder;
use index::Unchecked;
use meilisearch_types::settings::Unchecked;
use crate::{
reader::Document,
@ -332,7 +332,8 @@ pub(crate) mod test {
for (task, mut expected) in tasks_queue.lines().zip(create_test_tasks()) {
// TODO: This can be removed once `Duration` from the `TaskView` is implemented.
expected.0.duration = None;
assert_eq!(serde_json::from_str::<TaskView>(task).unwrap(), expected.0);
// TODO: uncomment this one once the we write the dump integration in the index-scheduler
// assert_eq!(serde_json::from_str::<TaskView>(task).unwrap(), expected.0);
if let Some(expected_update) = expected.1 {
let path = dump_path.join(format!("tasks/update_files/{}.jsonl", expected.0.uid));

2
index-scheduler/src/task.rs
View File

@ -416,6 +416,7 @@ pub enum Details {
dump_uid: String,
},
}
#[derive(Default, Debug, PartialEq, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DetailsView {
@ -441,6 +442,7 @@ pub struct DetailsView {
#[serde(flatten)]
settings: Option<Settings<Unchecked>>,
}
impl Details {
fn as_details_view(&self) -> DetailsView {
match self.clone() {