snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 02:27:40 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

dump the content of the dump tasks instead of recreating at import time with wrong API keys

Browse Source
This commit is contained in:
Tamo 2022-10-17 17:38:31 +02:00 committed by Clément Renault
parent 655705eb2b
commit 6bd6321226
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
6 changed files with 54 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
dump/src/lib.rs
View File

@ -1,8 +1,10 @@
use meilisearch_types::{ use meilisearch_types::{
error::ResponseError, error::ResponseError,
keys::Key,
milli::update::IndexDocumentsMethod, milli::update::IndexDocumentsMethod,
settings::Unchecked, settings::Unchecked,
tasks::{Details, KindWithContent, Status, Task, TaskId}, tasks::{Details, KindWithContent, Status, Task, TaskId},
InstanceUid,
}; };
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -120,6 +122,8 @@ pub enum KindDump {
}, },
DumpExport { DumpExport {
dump_uid: String, dump_uid: String,
keys: Vec<Key>,
instance_uid: Option<InstanceUid>,
}, },
Snapshot, Snapshot,
} }
@ -181,7 +185,15 @@ impl From<KindWithContent> for KindDump {
KindWithContent::TaskDeletion { query, tasks } => { KindWithContent::TaskDeletion { query, tasks } => {
KindDump::DeleteTasks { query, tasks } KindDump::DeleteTasks { query, tasks }
} }
KindWithContent::DumpExport { dump_uid, .. } => KindDump::DumpExport { dump_uid }, KindWithContent::DumpExport {
dump_uid,
keys,
instance_uid,
} => KindDump::DumpExport {
dump_uid,
keys,
instance_uid,
},
KindWithContent::Snapshot => KindDump::Snapshot, KindWithContent::Snapshot => KindDump::Snapshot,
} }
} }
@ -444,7 +456,7 @@ pub(crate) mod test {
drop(indexes); drop(indexes);
// ==== checking the task queue // ==== checking the task queue
for (task, expected) in dump.tasks().zip(create_test_tasks()) { for (task, expected) in dump.tasks().unwrap().zip(create_test_tasks()) {
let (task, content_file) = task.unwrap(); let (task, content_file) = task.unwrap();
assert_eq!(task, expected.0); assert_eq!(task, expected.0);
@ -463,7 +475,7 @@ pub(crate) mod test {
} }
// ==== checking the keys // ==== checking the keys
for (key, expected) in dump.keys().zip(create_test_api_keys()) { for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) {
assert_eq!(key.unwrap(), expected); assert_eq!(key.unwrap(), expected);
} }
} }

34
dump/src/reader/compat/v5_to_v6.rs
View File

@ -54,13 +54,16 @@ impl CompatV5ToV6 {
pub fn tasks( pub fn tasks(
&mut self, &mut self,
) -> Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_> { ) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
let instance_uid = self.instance_uid().ok().flatten().map(|uid| uid.clone());
let keys = self.keys()?.collect::<Result<Vec<_>>>()?;
let tasks = match self { let tasks = match self {
CompatV5ToV6::V5(v5) => v5.tasks(), CompatV5ToV6::V5(v5) => v5.tasks(),
CompatV5ToV6::Compat(compat) => compat.tasks(), CompatV5ToV6::Compat(compat) => compat.tasks(),
}; };
Box::new(tasks.map(|task| { Ok(Box::new(tasks.map(move |task| {
task.map(|(task, content_file)| { task.and_then(|(task, content_file)| {
let task_view: v5::tasks::TaskView = task.clone().into(); let task_view: v5::tasks::TaskView = task.clone().into();
let task = v6::Task { let task = v6::Task {
@ -116,9 +119,11 @@ impl CompatV5ToV6 {
allow_index_creation, allow_index_creation,
settings: settings.into(), settings: settings.into(),
}, },
v5::tasks::TaskContent::Dump { uid } => { v5::tasks::TaskContent::Dump { uid } => v6::Kind::DumpExport {
v6::Kind::DumpExport { dump_uid: uid } dump_uid: uid,
} keys: keys.clone(),
instance_uid: instance_uid.clone(),
},
}, },
details: task_view.details.map(|details| match details { details: task_view.details.map(|details| match details {
v5::Details::DocumentAddition { v5::Details::DocumentAddition {
@ -152,17 +157,18 @@ impl CompatV5ToV6 {
finished_at: task_view.finished_at, finished_at: task_view.finished_at,
}; };
(task, content_file) Ok((task, content_file))
}) })
})) })))
} }
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v6::Key>> + '_> { pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
let keys = match self { let keys = match self {
CompatV5ToV6::V5(v5) => v5.keys(), CompatV5ToV6::V5(v5) => v5.keys()?,
CompatV5ToV6::Compat(compat) => compat.keys(), CompatV5ToV6::Compat(compat) => compat.keys(),
}; };
Box::new(keys.map(|key| {
Ok(Box::new(keys.map(|key| {
key.map(|key| v6::Key { key.map(|key| v6::Key {
description: key.description, description: key.description,
name: key.name, name: key.name,
@ -186,7 +192,7 @@ impl CompatV5ToV6 {
created_at: key.created_at, created_at: key.created_at,
updated_at: key.updated_at, updated_at: key.updated_at,
}) })
})) })))
} }
} }
@ -412,7 +418,7 @@ pub(crate) mod test {
insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d"); insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
// tasks // tasks
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap(); let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8c6cd41457c0b7e4c6727c9c85b7abac"); meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8c6cd41457c0b7e4c6727c9c85b7abac");
assert_eq!(update_files.len(), 22); assert_eq!(update_files.len(), 22);
@ -421,7 +427,7 @@ pub(crate) mod test {
assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed
// keys // keys
let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap(); let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb"); meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb");
// indexes // indexes

8
dump/src/reader/mod.rs
View File

@ -98,16 +98,16 @@ impl DumpReader {
pub fn tasks( pub fn tasks(
&mut self, &mut self,
) -> Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_> { ) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
match self { match self {
DumpReader::Current(current) => current.tasks(), DumpReader::Current(current) => Ok(current.tasks()),
DumpReader::Compat(compat) => compat.tasks(), DumpReader::Compat(compat) => compat.tasks(),
} }
} }
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v6::Key>> + '_> { pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
match self { match self {
DumpReader::Current(current) => current.keys(), DumpReader::Current(current) => Ok(current.keys()),
DumpReader::Compat(compat) => compat.keys(), DumpReader::Compat(compat) => compat.keys(),
} }
} }

13
dump/src/reader/v5/mod.rs
View File

@ -34,7 +34,7 @@
use std::{ use std::{
fs::{self, File}, fs::{self, File},
io::{BufRead, BufReader}, io::{BufRead, BufReader, Seek, SeekFrom},
path::Path, path::Path,
}; };
@ -176,12 +176,11 @@ impl V5Reader {
})) }))
} }
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Key>> + '_> { pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<Key>> + '_>> {
Box::new( self.keys.seek(SeekFrom::Start(0))?;
(&mut self.keys) Ok(Box::new((&mut self.keys).lines().map(
.lines() |line| -> Result<_> { Ok(serde_json::from_str(&line?)?) },
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }), )))
)
} }
} }

12
index-scheduler/src/lib.rs
View File

@ -11,10 +11,8 @@ pub type TaskId = u32;
use dump::{KindDump, TaskDump, UpdateFile}; use dump::{KindDump, TaskDump, UpdateFile};
pub use error::Error; pub use error::Error;
use meilisearch_types::keys::Key;
use meilisearch_types::milli::documents::DocumentsBatchBuilder; use meilisearch_types::milli::documents::DocumentsBatchBuilder;
use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task}; use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
use meilisearch_types::InstanceUid;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
@ -400,8 +398,6 @@ impl IndexScheduler {
&mut self, &mut self,
task: TaskDump, task: TaskDump,
content_file: Option<Box<UpdateFile>>, content_file: Option<Box<UpdateFile>>,
keys: &[Key],
instance_uid: Option<InstanceUid>,
) -> Result<Task> { ) -> Result<Task> {
// Currently we don't need to access the tasks queue while loading a dump thus I can block everything. // Currently we don't need to access the tasks queue while loading a dump thus I can block everything.
let mut wtxn = self.env.write_txn()?; let mut wtxn = self.env.write_txn()?;
@ -479,9 +475,13 @@ impl IndexScheduler {
KindDump::DeleteTasks { query, tasks } => { KindDump::DeleteTasks { query, tasks } => {
KindWithContent::TaskDeletion { query, tasks } KindWithContent::TaskDeletion { query, tasks }
} }
KindDump::DumpExport { dump_uid } => KindWithContent::DumpExport { KindDump::DumpExport {
dump_uid, dump_uid,
keys: keys.to_vec(), keys,
instance_uid,
} => KindWithContent::DumpExport {
dump_uid,
keys,
instance_uid, instance_uid,
}, },
KindDump::Snapshot => KindWithContent::Snapshot, KindDump::Snapshot => KindWithContent::Snapshot,

6
meilisearch-http/src/lib.rs
View File

@ -190,7 +190,7 @@ fn import_dump(
// 2. Import the `Key`s. // 2. Import the `Key`s.
let mut keys = Vec::new(); let mut keys = Vec::new();
auth.raw_delete_all_keys()?; auth.raw_delete_all_keys()?;
for key in dump_reader.keys() { for key in dump_reader.keys()? {
let key = key?; let key = key?;
auth.raw_insert_key(key.clone())?; auth.raw_insert_key(key.clone())?;
keys.push(key); keys.push(key);
@ -259,9 +259,9 @@ fn import_dump(
} }
// 4. Import the tasks. // 4. Import the tasks.
for ret in dump_reader.tasks() { for ret in dump_reader.tasks()? {
let (task, file) = ret?; let (task, file) = ret?;
index_scheduler.register_dumped_task(task, file, &keys, instance_uid)?; index_scheduler.register_dumped_task(task, file)?;
} }
Ok(()) Ok(())
} }