snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 18:17:39 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

implement the dump v4 import

Browse Source
This commit is contained in:
Tamo 2022-10-06 15:49:30 +02:00 committed by Clément Renault
parent c50b44039e
commit a8128678a4
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
17 changed files with 3933 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
dump/src/reader/compat/v4.rs
View File

@ -1,145 +0,0 @@
use meilisearch_types::error::ResponseError;
use meilisearch_types::index_uid::IndexUid;
use milli::update::IndexDocumentsMethod;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use uuid::Uuid;
use crate::index::{Settings, Unchecked};
use crate::tasks::batch::BatchId;
use crate::tasks::task::{
DocumentDeletion, TaskContent as NewTaskContent, TaskEvent as NewTaskEvent, TaskId, TaskResult,
};
#[derive(Debug, Serialize, Deserialize)]
pub struct Task {
pub id: TaskId,
pub index_uid: IndexUid,
pub content: TaskContent,
pub events: Vec<TaskEvent>,
}
impl From<Task> for crate::tasks::task::Task {
fn from(other: Task) -> Self {
Self {
id: other.id,
content: NewTaskContent::from((other.index_uid, other.content)),
events: other.events.into_iter().map(Into::into).collect(),
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub enum TaskEvent {
Created(#[serde(with = "time::serde::rfc3339")] OffsetDateTime),
Batched {
#[serde(with = "time::serde::rfc3339")]
timestamp: OffsetDateTime,
batch_id: BatchId,
},
Processing(#[serde(with = "time::serde::rfc3339")] OffsetDateTime),
Succeded {
result: TaskResult,
#[serde(with = "time::serde::rfc3339")]
timestamp: OffsetDateTime,
},
Failed {
error: ResponseError,
#[serde(with = "time::serde::rfc3339")]
timestamp: OffsetDateTime,
},
}
impl From<TaskEvent> for NewTaskEvent {
fn from(other: TaskEvent) -> Self {
match other {
TaskEvent::Created(x) => NewTaskEvent::Created(x),
TaskEvent::Batched {
timestamp,
batch_id,
} => NewTaskEvent::Batched {
timestamp,
batch_id,
},
TaskEvent::Processing(x) => NewTaskEvent::Processing(x),
TaskEvent::Succeded { result, timestamp } => {
NewTaskEvent::Succeeded { result, timestamp }
}
TaskEvent::Failed { error, timestamp } => NewTaskEvent::Failed { error, timestamp },
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[allow(clippy::large_enum_variant)]
pub enum TaskContent {
DocumentAddition {
content_uuid: Uuid,
merge_strategy: IndexDocumentsMethod,
primary_key: Option<String>,
documents_count: usize,
allow_index_creation: bool,
},
DocumentDeletion(DocumentDeletion),
SettingsUpdate {
settings: Settings<Unchecked>,
/// Indicates whether the task was a deletion
is_deletion: bool,
allow_index_creation: bool,
},
IndexDeletion,
IndexCreation {
primary_key: Option<String>,
},
IndexUpdate {
primary_key: Option<String>,
},
Dump {
uid: String,
},
}
impl From<(IndexUid, TaskContent)> for NewTaskContent {
fn from((index_uid, content): (IndexUid, TaskContent)) -> Self {
match content {
TaskContent::DocumentAddition {
content_uuid,
merge_strategy,
primary_key,
documents_count,
allow_index_creation,
} => NewTaskContent::DocumentAddition {
index_uid,
content_uuid,
merge_strategy,
primary_key,
documents_count,
allow_index_creation,
},
TaskContent::DocumentDeletion(deletion) => NewTaskContent::DocumentDeletion {
index_uid,
deletion,
},
TaskContent::SettingsUpdate {
settings,
is_deletion,
allow_index_creation,
} => NewTaskContent::SettingsUpdate {
index_uid,
settings,
is_deletion,
allow_index_creation,
},
TaskContent::IndexDeletion => NewTaskContent::IndexDeletion { index_uid },
TaskContent::IndexCreation { primary_key } => NewTaskContent::IndexCreation {
index_uid,
primary_key,
},
TaskContent::IndexUpdate { primary_key } => NewTaskContent::IndexUpdate {
index_uid,
primary_key,
},
TaskContent::Dump { uid } => NewTaskContent::Dump { uid },
}
}
}

77
dump/src/reader/v4/keys.rs Normal file
View File

@ -0,0 +1,77 @@
use serde::Deserialize;
use time::OffsetDateTime;
pub const KEY_ID_LENGTH: usize = 8;
pub type KeyId = [u8; KEY_ID_LENGTH];
#[derive(Debug, Deserialize)]
#[cfg_attr(test, derive(serde::Serialize))]
pub struct Key {
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
pub id: KeyId,
pub actions: Vec<Action>,
pub indexes: Vec<String>,
#[serde(with = "time::serde::rfc3339::option")]
pub expires_at: Option<OffsetDateTime>,
#[serde(with = "time::serde::rfc3339")]
pub created_at: OffsetDateTime,
#[serde(with = "time::serde::rfc3339")]
pub updated_at: OffsetDateTime,
}
#[derive(Copy, Clone, Deserialize, Debug, Eq, PartialEq)]
#[cfg_attr(test, derive(serde::Serialize))]
#[repr(u8)]
pub enum Action {
#[serde(rename = "*")]
All = 0,
#[serde(rename = "search")]
Search = actions::SEARCH,
#[serde(rename = "documents.add")]
DocumentsAdd = actions::DOCUMENTS_ADD,
#[serde(rename = "documents.get")]
DocumentsGet = actions::DOCUMENTS_GET,
#[serde(rename = "documents.delete")]
DocumentsDelete = actions::DOCUMENTS_DELETE,
#[serde(rename = "indexes.create")]
IndexesAdd = actions::INDEXES_CREATE,
#[serde(rename = "indexes.get")]
IndexesGet = actions::INDEXES_GET,
#[serde(rename = "indexes.update")]
IndexesUpdate = actions::INDEXES_UPDATE,
#[serde(rename = "indexes.delete")]
IndexesDelete = actions::INDEXES_DELETE,
#[serde(rename = "tasks.get")]
TasksGet = actions::TASKS_GET,
#[serde(rename = "settings.get")]
SettingsGet = actions::SETTINGS_GET,
#[serde(rename = "settings.update")]
SettingsUpdate = actions::SETTINGS_UPDATE,
#[serde(rename = "stats.get")]
StatsGet = actions::STATS_GET,
#[serde(rename = "dumps.create")]
DumpsCreate = actions::DUMPS_CREATE,
#[serde(rename = "dumps.get")]
DumpsGet = actions::DUMPS_GET,
#[serde(rename = "version")]
Version = actions::VERSION,
}
pub mod actions {
pub const SEARCH: u8 = 1;
pub const DOCUMENTS_ADD: u8 = 2;
pub const DOCUMENTS_GET: u8 = 3;
pub const DOCUMENTS_DELETE: u8 = 4;
pub const INDEXES_CREATE: u8 = 5;
pub const INDEXES_GET: u8 = 6;
pub const INDEXES_UPDATE: u8 = 7;
pub const INDEXES_DELETE: u8 = 8;
pub const TASKS_GET: u8 = 9;
pub const SETTINGS_GET: u8 = 10;
pub const SETTINGS_UPDATE: u8 = 11;
pub const STATS_GET: u8 = 12;
pub const DUMPS_CREATE: u8 = 13;
pub const DUMPS_GET: u8 = 14;
pub const VERSION: u8 = 15;
}

142
dump/src/reader/v4/meta.rs Normal file
View File

@ -0,0 +1,142 @@
use std::{
fmt::{self, Display, Formatter},
marker::PhantomData,
str::FromStr,
};
use serde::{de::Visitor, Deserialize, Deserializer};
use uuid::Uuid;
use super::settings::{Settings, Unchecked};
#[derive(Deserialize, Debug)]
#[cfg_attr(test, derive(serde::Serialize))]
pub struct IndexUuid {
pub uid: String,
pub index_meta: IndexMeta,
}
#[derive(Deserialize, Debug)]
#[cfg_attr(test, derive(serde::Serialize))]
pub struct IndexMeta {
pub uuid: Uuid,
pub creation_task_id: usize,
}
// There is one in each indexes under `meta.json`.
#[derive(Deserialize)]
#[cfg_attr(test, derive(serde::Serialize))]
pub struct DumpMeta {
pub settings: Settings<Unchecked>,
pub primary_key: Option<String>,
}
#[derive(Deserialize, Debug, Clone, PartialEq, Eq)]
#[cfg_attr(test, derive(serde::Serialize))]
pub struct IndexUid(pub String);
impl TryFrom<String> for IndexUid {
type Error = IndexUidFormatError;
fn try_from(uid: String) -> Result<Self, Self::Error> {
if !uid
.chars()
.all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
|| uid.is_empty()
|| uid.len() > 400
{
Err(IndexUidFormatError { invalid_uid: uid })
} else {
Ok(IndexUid(uid))
}
}
}
impl FromStr for IndexUid {
type Err = IndexUidFormatError;
fn from_str(uid: &str) -> Result<IndexUid, IndexUidFormatError> {
uid.to_string().try_into()
}
}
impl From<IndexUid> for String {
fn from(uid: IndexUid) -> Self {
uid.into_inner()
}
}
#[derive(Debug)]
pub struct IndexUidFormatError {
pub invalid_uid: String,
}
impl Display for IndexUidFormatError {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
write!(
f,
"invalid index uid `{}`, the uid must be an integer \
or a string containing only alphanumeric characters \
a-z A-Z 0-9, hyphens - and underscores _.",
self.invalid_uid,
)
}
}
impl std::error::Error for IndexUidFormatError {}
/// A type that tries to match either a star (*) or
/// any other thing that implements `FromStr`.
#[derive(Debug)]
#[cfg_attr(test, derive(serde::Serialize))]
pub enum StarOr<T> {
Star,
Other(T),
}
impl<'de, T, E> Deserialize<'de> for StarOr<T>
where
T: FromStr<Err = E>,
E: Display,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
/// Serde can't differentiate between `StarOr::Star` and `StarOr::Other` without a tag.
/// Simply using `#[serde(untagged)]` + `#[serde(rename="*")]` will lead to attempting to
/// deserialize everything as a `StarOr::Other`, including "*".
/// [`#[serde(other)]`](https://serde.rs/variant-attrs.html#other) might have helped but is
/// not supported on untagged enums.
struct StarOrVisitor<T>(PhantomData<T>);
impl<'de, T, FE> Visitor<'de> for StarOrVisitor<T>
where
T: FromStr<Err = FE>,
FE: Display,
{
type Value = StarOr<T>;
fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
formatter.write_str("a string")
}
fn visit_str<SE>(self, v: &str) -> Result<Self::Value, SE>
where
SE: serde::de::Error,
{
match v {
"*" => Ok(StarOr::Star),
v => {
let other = FromStr::from_str(v).map_err(|e: T::Err| {
SE::custom(format!("Invalid `other` value: {}", e))
})?;
Ok(StarOr::Other(other))
}
}
}
}
deserializer.deserialize_str(StarOrVisitor(PhantomData))
}
}

277
dump/src/reader/v4/mod.rs
View File

@ -1 +1,276 @@
// hello
use std::{
fs::{self, File},
io::{BufRead, BufReader},
path::Path,
};
use serde::{Deserialize, Serialize};
use tempfile::TempDir;
use time::OffsetDateTime;
use uuid::Uuid;
mod keys;
mod meta;
mod settings;
mod tasks;
use crate::{IndexMetadata, Result, Version};
use self::{
keys::Key,
meta::{DumpMeta, IndexUuid},
settings::{Checked, Settings},
tasks::Task,
};
use super::{/* compat::v4_to_v5::CompatV4ToV5, */ DumpReader, IndexReader};
pub type Document = serde_json::Map<String, serde_json::Value>;
pub type UpdateFile = File;
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Metadata {
db_version: String,
index_db_size: usize,
update_db_size: usize,
#[serde(with = "time::serde::rfc3339")]
dump_date: OffsetDateTime,
}
pub struct V4Reader {
dump: TempDir,
metadata: Metadata,
tasks: BufReader<File>,
keys: BufReader<File>,
index_uuid: Vec<IndexUuid>,
}
impl V4Reader {
pub fn open(dump: TempDir) -> Result<Self> {
let meta_file = fs::read(dump.path().join("metadata.json"))?;
let metadata = serde_json::from_reader(&*meta_file)?;
let index_uuid = File::open(dump.path().join("index_uuids/data.jsonl"))?;
let index_uuid = BufReader::new(index_uuid);
let index_uuid = index_uuid
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })
.collect::<Result<Vec<_>>>()?;
Ok(V4Reader {
metadata,
tasks: BufReader::new(
File::open(dump.path().join("updates").join("data.jsonl")).unwrap(),
),
keys: BufReader::new(File::open(dump.path().join("keys"))?),
index_uuid,
dump,
})
}
// pub fn to_v5(self) -> CompatV4ToV5 {
// CompatV4ToV5::new(self)
// }
pub fn version(&self) -> Version {
Version::V4
}
pub fn date(&self) -> Option<OffsetDateTime> {
Some(self.metadata.dump_date)
}
pub fn instance_uid(&self) -> Result<Option<Uuid>> {
let uuid = fs::read_to_string(self.dump.path().join("instance-uid"))?;
Ok(Some(Uuid::parse_str(&uuid)?))
}
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V4IndexReader>> + '_> {
Ok(self.index_uuid.iter().map(|index| -> Result<_> {
Ok(V4IndexReader::new(
index.uid.clone(),
&self
.dump
.path()
.join("indexes")
.join(index.index_meta.uuid.to_string()),
)?)
}))
}
pub fn tasks(&mut self) -> impl Iterator<Item = Result<(Task, Option<UpdateFile>)>> + '_ {
(&mut self.tasks).lines().map(|line| -> Result<_> {
let task: Task = serde_json::from_str(&line?)?;
if !task.is_finished() {
if let Some(uuid) = task.get_content_uuid() {
let update_file_path = self
.dump
.path()
.join("updates")
.join("updates_files")
.join(uuid.to_string());
Ok((task, Some(File::open(update_file_path).unwrap())))
} else {
Ok((task, None))
}
} else {
Ok((task, None))
}
})
}
pub fn keys(&mut self) -> impl Iterator<Item = Result<Key>> + '_ {
(&mut self.keys)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })
}
}
pub struct V4IndexReader {
metadata: IndexMetadata,
settings: Settings<Checked>,
documents: BufReader<File>,
}
impl V4IndexReader {
pub fn new(name: String, path: &Path) -> Result<Self> {
let meta = File::open(path.join("meta.json"))?;
let meta: DumpMeta = serde_json::from_reader(meta)?;
let metadata = IndexMetadata {
uid: name,
primary_key: meta.primary_key,
// FIXME: Iterate over the whole task queue to find the creation and last update date.
created_at: OffsetDateTime::now_utc(),
updated_at: OffsetDateTime::now_utc(),
};
let ret = V4IndexReader {
metadata,
settings: meta.settings.check(),
documents: BufReader::new(File::open(path.join("documents.jsonl"))?),
};
Ok(ret)
}
pub fn metadata(&self) -> &IndexMetadata {
&self.metadata
}
pub fn documents(&mut self) -> Result<impl Iterator<Item = Result<Document>> + '_> {
Ok((&mut self.documents)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }))
}
pub fn settings(&mut self) -> Result<Settings<Checked>> {
Ok(self.settings.clone())
}
}
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
use super::*;
#[test]
fn read_dump_v4() {
let dump = File::open("tests/assets/v4.dump").unwrap();
let dir = TempDir::new().unwrap();
let mut dump = BufReader::new(dump);
let gz = GzDecoder::new(&mut dump);
let mut archive = tar::Archive::new(gz);
archive.unpack(dir.path()).unwrap();
let mut dump = V4Reader::open(dir).unwrap();
// top level infos
insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
// tasks
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
insta::assert_json_snapshot!(tasks);
assert_eq!(update_files.len(), 10);
assert!(update_files[0].is_some()); // the enqueued document addition
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
// keys
let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap();
insta::assert_json_snapshot!(keys);
// indexes
let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
// the index are not ordered in any way by default
indexes.sort_by_key(|index| index.metadata().uid.to_string());
let mut products = indexes.pop().unwrap();
let mut movies = indexes.pop().unwrap();
let mut spells = indexes.pop().unwrap();
assert!(indexes.is_empty());
// products
insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
{
"uid": "products",
"primaryKey": "sku",
"createdAt": "[now]",
"updatedAt": "[now]"
}
"###);
insta::assert_debug_snapshot!(products.settings());
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
assert_eq!(documents.len(), 10);
insta::assert_json_snapshot!(documents);
// movies
insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
{
"uid": "movies",
"primaryKey": "id",
"createdAt": "[now]",
"updatedAt": "[now]"
}
"###);
insta::assert_debug_snapshot!(movies.settings());
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
assert_eq!(documents.len(), 110);
insta::assert_debug_snapshot!(documents);
// spells
insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
{
"uid": "dnd_spells",
"primaryKey": "index",
"createdAt": "[now]",
"updatedAt": "[now]"
}
"###);
insta::assert_debug_snapshot!(spells.settings());
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
assert_eq!(documents.len(), 10);
insta::assert_json_snapshot!(documents);
}
}

266
dump/src/reader/v4/settings.rs Normal file
View File

@ -0,0 +1,266 @@
use std::{
collections::{BTreeMap, BTreeSet},
marker::PhantomData,
num::NonZeroUsize,
};
use serde::{Deserialize, Deserializer};
#[cfg(test)]
fn serialize_with_wildcard<S>(
field: &Setting<Vec<String>>,
s: S,
) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
use serde::Serialize;
let wildcard = vec!["*".to_string()];
match field {
Setting::Set(value) => Some(value),
Setting::Reset => Some(&wildcard),
Setting::NotSet => None,
}
.serialize(s)
}
#[derive(Clone, Default, Debug, PartialEq)]
#[cfg_attr(test, derive(serde::Serialize))]
pub struct Checked;
#[derive(Clone, Default, Debug, Deserialize, PartialEq)]
#[cfg_attr(test, derive(serde::Serialize))]
pub struct Unchecked;
#[derive(Debug, Clone, Default, Deserialize, PartialEq)]
#[cfg_attr(test, derive(serde::Serialize))]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
pub struct MinWordSizeTyposSetting {
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub one_typo: Setting<u8>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub two_typos: Setting<u8>,
}
#[derive(Debug, Clone, Default, Deserialize, PartialEq)]
#[cfg_attr(test, derive(serde::Serialize))]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
pub struct TypoSettings {
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub enabled: Setting<bool>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub min_word_size_for_typos: Setting<MinWordSizeTyposSetting>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub disable_on_words: Setting<BTreeSet<String>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub disable_on_attributes: Setting<BTreeSet<String>>,
}
/// Holds all the settings for an index. `T` can either be `Checked` if they represents settings
/// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a
/// call to `check` will return a `Settings<Checked>` from a `Settings<Unchecked>`.
#[derive(Debug, Clone, Default, Deserialize, PartialEq)]
#[cfg_attr(test, derive(serde::Serialize))]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
#[serde(bound(
serialize = "T: serde::Serialize",
deserialize = "T: Deserialize<'static>"
))]
pub struct Settings<T> {
#[serde(
default,
serialize_with = "serialize_with_wildcard",
skip_serializing_if = "Setting::is_not_set"
)]
pub displayed_attributes: Setting<Vec<String>>,
#[serde(
default,
serialize_with = "serialize_with_wildcard",
skip_serializing_if = "Setting::is_not_set"
)]
pub searchable_attributes: Setting<Vec<String>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub filterable_attributes: Setting<BTreeSet<String>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub sortable_attributes: Setting<BTreeSet<String>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub ranking_rules: Setting<Vec<String>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub stop_words: Setting<BTreeSet<String>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub synonyms: Setting<BTreeMap<String, Vec<String>>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub distinct_attribute: Setting<String>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
pub typo_tolerance: Setting<TypoSettings>,
#[serde(skip)]
pub _kind: PhantomData<T>,
}
impl Settings<Checked> {
pub fn cleared() -> Settings<Checked> {
Settings {
displayed_attributes: Setting::Reset,
searchable_attributes: Setting::Reset,
filterable_attributes: Setting::Reset,
sortable_attributes: Setting::Reset,
ranking_rules: Setting::Reset,
stop_words: Setting::Reset,
synonyms: Setting::Reset,
distinct_attribute: Setting::Reset,
typo_tolerance: Setting::Reset,
_kind: PhantomData,
}
}
pub fn into_unchecked(self) -> Settings<Unchecked> {
let Self {
displayed_attributes,
searchable_attributes,
filterable_attributes,
sortable_attributes,
ranking_rules,
stop_words,
synonyms,
distinct_attribute,
typo_tolerance,
..
} = self;
Settings {
displayed_attributes,
searchable_attributes,
filterable_attributes,
sortable_attributes,
ranking_rules,
stop_words,
synonyms,
distinct_attribute,
typo_tolerance,
_kind: PhantomData,
}
}
}
impl Settings<Unchecked> {
pub fn check(self) -> Settings<Checked> {
let displayed_attributes = match self.displayed_attributes {
Setting::Set(fields) => {
if fields.iter().any(|f| f == "*") {
Setting::Reset
} else {
Setting::Set(fields)
}
}
otherwise => otherwise,
};
let searchable_attributes = match self.searchable_attributes {
Setting::Set(fields) => {
if fields.iter().any(|f| f == "*") {
Setting::Reset
} else {
Setting::Set(fields)
}
}
otherwise => otherwise,
};
Settings {
displayed_attributes,
searchable_attributes,
filterable_attributes: self.filterable_attributes,
sortable_attributes: self.sortable_attributes,
ranking_rules: self.ranking_rules,
stop_words: self.stop_words,
synonyms: self.synonyms,
distinct_attribute: self.distinct_attribute,
typo_tolerance: self.typo_tolerance,
_kind: PhantomData,
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[cfg_attr(test, derive(serde::Serialize))]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
pub struct Facets {
pub level_group_size: Option<NonZeroUsize>,
pub min_level_size: Option<NonZeroUsize>,
}
#[derive(Debug, Clone, PartialEq, Copy)]
pub enum Setting<T> {
Set(T),
Reset,
NotSet,
}
impl<T> Default for Setting<T> {
fn default() -> Self {
Self::NotSet
}
}
impl<T> Setting<T> {
pub fn set(self) -> Option<T> {
match self {
Self::Set(value) => Some(value),
_ => None,
}
}
pub const fn as_ref(&self) -> Setting<&T> {
match *self {
Self::Set(ref value) => Setting::Set(value),
Self::Reset => Setting::Reset,
Self::NotSet => Setting::NotSet,
}
}
pub const fn is_not_set(&self) -> bool {
matches!(self, Self::NotSet)
}
/// If `Self` is `Reset`, then map self to `Set` with the provided `val`.
pub fn or_reset(self, val: T) -> Self {
match self {
Self::Reset => Self::Set(val),
otherwise => otherwise,
}
}
}
#[cfg(test)]
impl<T: serde::Serialize> serde::Serialize for Setting<T> {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
Self::Set(value) => Some(value),
// Usually not_set isn't serialized by setting skip_serializing_if field attribute
Self::NotSet | Self::Reset => None,
}
.serialize(serializer)
}
}
impl<'de, T: Deserialize<'de>> Deserialize<'de> for Setting<T> {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Deserialize::deserialize(deserializer).map(|x| match x {
Some(x) => Self::Set(x),
None => Self::Reset, // Reset is forced by sending null value
})
}
}

1252
dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-10.snap Normal file
View File

File diff suppressed because it is too large Load Diff

57
dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-12.snap Normal file
View File

@ -0,0 +1,57 @@
---
source: dump/src/reader/v4/mod.rs
expression: spells.settings()
---
Ok(
Settings {
displayed_attributes: Reset,
searchable_attributes: Reset,
filterable_attributes: Set(
{},
),
sortable_attributes: Set(
{},
),
ranking_rules: Set(
[
"words",
"typo",
"proximity",
"attribute",
"sort",
"exactness",
],
),
stop_words: Set(
{},
),
synonyms: Set(
{},
),
distinct_attribute: Reset,
typo_tolerance: Set(
TypoSettings {
enabled: Set(
true,
),
min_word_size_for_typos: Set(
MinWordSizeTyposSetting {
one_typo: Set(
5,
),
two_typos: Set(
9,
),
},
),
disable_on_words: Set(
{},
),
disable_on_attributes: Set(
{},
),
},
),
_kind: PhantomData<dump::reader::v4::settings::Checked>,
},
)

533
dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-13.snap Normal file
View File

@ -0,0 +1,533 @@
---
source: dump/src/reader/v4/mod.rs
expression: documents
---
[
{
"index": "acid-arrow",
"name": "Acid Arrow",
"desc": [
"A shimmering green arrow streaks toward a target within range and bursts in a spray of acid. Make a ranged spell attack against the target. On a hit, the target takes 4d4 acid damage immediately and 2d4 acid damage at the end of its next turn. On a miss, the arrow splashes the target with acid for half as much of the initial damage and no damage at the end of its next turn."
],
"higher_level": [
"When you cast this spell using a spell slot of 3rd level or higher, the damage (both initial and later) increases by 1d4 for each slot level above 2nd."
],
"range": "90 feet",
"components": [
"V",
"S",
"M"
],
"material": "Powdered rhubarb leaf and an adder's stomach.",
"ritual": false,
"duration": "Instantaneous",
"concentration": false,
"casting_time": "1 action",
"level": 2,
"attack_type": "ranged",
"damage": {
"damage_type": {
"index": "acid",
"name": "Acid",
"url": "/api/damage-types/acid"
},
"damage_at_slot_level": {
"2": "4d4",
"3": "5d4",
"4": "6d4",
"5": "7d4",
"6": "8d4",
"7": "9d4",
"8": "10d4",
"9": "11d4"
}
},
"school": {
"index": "evocation",
"name": "Evocation",
"url": "/api/magic-schools/evocation"
},
"classes": [
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
},
{
"index": "land",
"name": "Land",
"url": "/api/subclasses/land"
}
],
"url": "/api/spells/acid-arrow"
},
{
"index": "acid-splash",
"name": "Acid Splash",
"desc": [
"You hurl a bubble of acid. Choose one creature within range, or choose two creatures within range that are within 5 feet of each other. A target must succeed on a dexterity saving throw or take 1d6 acid damage.",
"This spell's damage increases by 1d6 when you reach 5th level (2d6), 11th level (3d6), and 17th level (4d6)."
],
"range": "60 feet",
"components": [
"V",
"S"
],
"ritual": false,
"duration": "Instantaneous",
"concentration": false,
"casting_time": "1 action",
"level": 0,
"damage": {
"damage_type": {
"index": "acid",
"name": "Acid",
"url": "/api/damage-types/acid"
},
"damage_at_character_level": {
"1": "1d6",
"5": "2d6",
"11": "3d6",
"17": "4d6"
}
},
"school": {
"index": "conjuration",
"name": "Conjuration",
"url": "/api/magic-schools/conjuration"
},
"classes": [
{
"index": "sorcerer",
"name": "Sorcerer",
"url": "/api/classes/sorcerer"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/acid-splash",
"dc": {
"dc_type": {
"index": "dex",
"name": "DEX",
"url": "/api/ability-scores/dex"
},
"dc_success": "none"
}
},
{
"index": "aid",
"name": "Aid",
"desc": [
"Your spell bolsters your allies with toughness and resolve. Choose up to three creatures within range. Each target's hit point maximum and current hit points increase by 5 for the duration."
],
"higher_level": [
"When you cast this spell using a spell slot of 3rd level or higher, a target's hit points increase by an additional 5 for each slot level above 2nd."
],
"range": "30 feet",
"components": [
"V",
"S",
"M"
],
"material": "A tiny strip of white cloth.",
"ritual": false,
"duration": "8 hours",
"concentration": false,
"casting_time": "1 action",
"level": 2,
"school": {
"index": "abjuration",
"name": "Abjuration",
"url": "/api/magic-schools/abjuration"
},
"classes": [
{
"index": "cleric",
"name": "Cleric",
"url": "/api/classes/cleric"
},
{
"index": "paladin",
"name": "Paladin",
"url": "/api/classes/paladin"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/aid",
"heal_at_slot_level": {
"2": "5",
"3": "10",
"4": "15",
"5": "20",
"6": "25",
"7": "30",
"8": "35",
"9": "40"
}
},
{
"index": "alarm",
"name": "Alarm",
"desc": [
"You set an alarm against unwanted intrusion. Choose a door, a window, or an area within range that is no larger than a 20-foot cube. Until the spell ends, an alarm alerts you whenever a Tiny or larger creature touches or enters the warded area. When you cast the spell, you can designate creatures that won't set off the alarm. You also choose whether the alarm is mental or audible.",
"A mental alarm alerts you with a ping in your mind if you are within 1 mile of the warded area. This ping awakens you if you are sleeping.",
"An audible alarm produces the sound of a hand bell for 10 seconds within 60 feet."
],
"range": "30 feet",
"components": [
"V",
"S",
"M"
],
"material": "A tiny bell and a piece of fine silver wire.",
"ritual": true,
"duration": "8 hours",
"concentration": false,
"casting_time": "1 minute",
"level": 1,
"school": {
"index": "abjuration",
"name": "Abjuration",
"url": "/api/magic-schools/abjuration"
},
"classes": [
{
"index": "ranger",
"name": "Ranger",
"url": "/api/classes/ranger"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/alarm",
"area_of_effect": {
"type": "cube",
"size": 20
}
},
{
"index": "alter-self",
"name": "Alter Self",
"desc": [
"You assume a different form. When you cast the spell, choose one of the following options, the effects of which last for the duration of the spell. While the spell lasts, you can end one option as an action to gain the benefits of a different one.",
"***Aquatic Adaptation.*** You adapt your body to an aquatic environment, sprouting gills and growing webbing between your fingers. You can breathe underwater and gain a swimming speed equal to your walking speed.",
"***Change Appearance.*** You transform your appearance. You decide what you look like, including your height, weight, facial features, sound of your voice, hair length, coloration, and distinguishing characteristics, if any. You can make yourself appear as a member of another race, though none of your statistics change. You also can't appear as a creature of a different size than you, and your basic shape stays the same; if you're bipedal, you can't use this spell to become quadrupedal, for instance. At any time for the duration of the spell, you can use your action to change your appearance in this way again.",
"***Natural Weapons.*** You grow claws, fangs, spines, horns, or a different natural weapon of your choice. Your unarmed strikes deal 1d6 bludgeoning, piercing, or slashing damage, as appropriate to the natural weapon you chose, and you are proficient with your unarmed strikes. Finally, the natural weapon is magic and you have a +1 bonus to the attack and damage rolls you make using it."
],
"range": "Self",
"components": [
"V",
"S"
],
"ritual": false,
"duration": "Up to 1 hour",
"concentration": true,
"casting_time": "1 action",
"level": 2,
"school": {
"index": "transmutation",
"name": "Transmutation",
"url": "/api/magic-schools/transmutation"
},
"classes": [
{
"index": "sorcerer",
"name": "Sorcerer",
"url": "/api/classes/sorcerer"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/alter-self"
},
{
"index": "animal-friendship",
"name": "Animal Friendship",
"desc": [
"This spell lets you convince a beast that you mean it no harm. Choose a beast that you can see within range. It must see and hear you. If the beast's Intelligence is 4 or higher, the spell fails. Otherwise, the beast must succeed on a wisdom saving throw or be charmed by you for the spell's duration. If you or one of your companions harms the target, the spells ends."
],
"range": "30 feet",
"components": [
"V",
"S",
"M"
],
"material": "A morsel of food.",
"ritual": false,
"duration": "24 hours",
"concentration": false,
"casting_time": "1 action",
"level": 1,
"school": {
"index": "enchantment",
"name": "Enchantment",
"url": "/api/magic-schools/enchantment"
},
"classes": [
{
"index": "bard",
"name": "Bard",
"url": "/api/classes/bard"
},
{
"index": "cleric",
"name": "Cleric",
"url": "/api/classes/cleric"
},
{
"index": "druid",
"name": "Druid",
"url": "/api/classes/druid"
},
{
"index": "ranger",
"name": "Ranger",
"url": "/api/classes/ranger"
}
],
"subclasses": [],
"url": "/api/spells/animal-friendship",
"dc": {
"dc_type": {
"index": "wis",
"name": "WIS",
"url": "/api/ability-scores/wis"
},
"dc_success": "none"
}
},
{
"index": "animal-messenger",
"name": "Animal Messenger",
"desc": [
"By means of this spell, you use an animal to deliver a message. Choose a Tiny beast you can see within range, such as a squirrel, a blue jay, or a bat. You specify a location, which you must have visited, and a recipient who matches a general description, such as \"a man or woman dressed in the uniform of the town guard\" or \"a red-haired dwarf wearing a pointed hat.\" You also speak a message of up to twenty-five words. The target beast travels for the duration of the spell toward the specified location, covering about 50 miles per 24 hours for a flying messenger, or 25 miles for other animals.",
"When the messenger arrives, it delivers your message to the creature that you described, replicating the sound of your voice. The messenger speaks only to a creature matching the description you gave. If the messenger doesn't reach its destination before the spell ends, the message is lost, and the beast makes its way back to where you cast this spell."
],
"higher_level": [
"If you cast this spell using a spell slot of 3nd level or higher, the duration of the spell increases by 48 hours for each slot level above 2nd."
],
"range": "30 feet",
"components": [
"V",
"S",
"M"
],
"material": "A morsel of food.",
"ritual": true,
"duration": "24 hours",
"concentration": false,
"casting_time": "1 action",
"level": 2,
"school": {
"index": "enchantment",
"name": "Enchantment",
"url": "/api/magic-schools/enchantment"
},
"classes": [
{
"index": "bard",
"name": "Bard",
"url": "/api/classes/bard"
},
{
"index": "druid",
"name": "Druid",
"url": "/api/classes/druid"
},
{
"index": "ranger",
"name": "Ranger",
"url": "/api/classes/ranger"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/animal-messenger"
},
{
"index": "animal-shapes",
"name": "Animal Shapes",
"desc": [
"Your magic turns others into beasts. Choose any number of willing creatures that you can see within range. You transform each target into the form of a Large or smaller beast with a challenge rating of 4 or lower. On subsequent turns, you can use your action to transform affected creatures into new forms.",
"The transformation lasts for the duration for each target, or until the target drops to 0 hit points or dies. You can choose a different form for each target. A target's game statistics are replaced by the statistics of the chosen beast, though the target retains its alignment and Intelligence, Wisdom, and Charisma scores. The target assumes the hit points of its new form, and when it reverts to its normal form, it returns to the number of hit points it had before it transformed. If it reverts as a result of dropping to 0 hit points, any excess damage carries over to its normal form. As long as the excess damage doesn't reduce the creature's normal form to 0 hit points, it isn't knocked unconscious. The creature is limited in the actions it can perform by the nature of its new form, and it can't speak or cast spells.",
"The target's gear melds into the new form. The target can't activate, wield, or otherwise benefit from any of its equipment."
],
"range": "30 feet",
"components": [
"V",
"S"
],
"ritual": false,
"duration": "Up to 24 hours",
"concentration": true,
"casting_time": "1 action",
"level": 8,
"school": {
"index": "transmutation",
"name": "Transmutation",
"url": "/api/magic-schools/transmutation"
},
"classes": [
{
"index": "druid",
"name": "Druid",
"url": "/api/classes/druid"
}
],
"subclasses": [],
"url": "/api/spells/animal-shapes"
},
{
"index": "animate-dead",
"name": "Animate Dead",
"desc": [
"This spell creates an undead servant. Choose a pile of bones or a corpse of a Medium or Small humanoid within range. Your spell imbues the target with a foul mimicry of life, raising it as an undead creature. The target becomes a skeleton if you chose bones or a zombie if you chose a corpse (the DM has the creature's game statistics).",
"On each of your turns, you can use a bonus action to mentally command any creature you made with this spell if the creature is within 60 feet of you (if you control multiple creatures, you can command any or all of them at the same time, issuing the same command to each one). You decide what action the creature will take and where it will move during its next turn, or you can issue a general command, such as to guard a particular chamber or corridor. If you issue no commands, the creature only defends itself against hostile creatures. Once given an order, the creature continues to follow it until its task is complete.",
"The creature is under your control for 24 hours, after which it stops obeying any command you've given it. To maintain control of the creature for another 24 hours, you must cast this spell on the creature again before the current 24-hour period ends. This use of the spell reasserts your control over up to four creatures you have animated with this spell, rather than animating a new one."
],
"higher_level": [
"When you cast this spell using a spell slot of 4th level or higher, you animate or reassert control over two additional undead creatures for each slot level above 3rd. Each of the creatures must come from a different corpse or pile of bones."
],
"range": "10 feet",
"components": [
"V",
"S",
"M"
],
"material": "A drop of blood, a piece of flesh, and a pinch of bone dust.",
"ritual": false,
"duration": "Instantaneous",
"concentration": false,
"casting_time": "1 minute",
"level": 3,
"school": {
"index": "necromancy",
"name": "Necromancy",
"url": "/api/magic-schools/necromancy"
},
"classes": [
{
"index": "cleric",
"name": "Cleric",
"url": "/api/classes/cleric"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/animate-dead"
},
{
"index": "animate-objects",
"name": "Animate Objects",
"desc": [
"Objects come to life at your command. Choose up to ten nonmagical objects within range that are not being worn or carried. Medium targets count as two objects, Large targets count as four objects, Huge targets count as eight objects. You can't animate any object larger than Huge. Each target animates and becomes a creature under your control until the spell ends or until reduced to 0 hit points.",
"As a bonus action, you can mentally command any creature you made with this spell if the creature is within 500 feet of you (if you control multiple creatures, you can command any or all of them at the same time, issuing the same command to each one). You decide what action the creature will take and where it will move during its next turn, or you can issue a general command, such as to guard a particular chamber or corridor. If you issue no commands, the creature only defends itself against hostile creatures. Once given an order, the creature continues to follow it until its task is complete.",
"##### Animated Object Statistics",
"| Size | HP | AC | Attack | Str | Dex |",
"|---|---|---|---|---|---|",
"| Tiny | 20 | 18 | +8 to hit, 1d4 + 4 damage | 4 | 18 |",
"| Small | 25 | 16 | +6 to hit, 1d8 + 2 damage | 6 | 14 |",
"| Medium | 40 | 13 | +5 to hit, 2d6 + 1 damage | 10 | 12 |",
"| Large | 50 | 10 | +6 to hit, 2d10 + 2 damage | 14 | 10 |",
"| Huge | 80 | 10 | +8 to hit, 2d12 + 4 damage | 18 | 6 |",
"An animated object is a construct with AC, hit points, attacks, Strength, and Dexterity determined by its size. Its Constitution is 10 and its Intelligence and Wisdom are 3, and its Charisma is 1. Its speed is 30 feet; if the object lacks legs or other appendages it can use for locomotion, it instead has a flying speed of 30 feet and can hover. If the object is securely attached to a surface or a larger object, such as a chain bolted to a wall, its speed is 0. It has blindsight with a radius of 30 feet and is blind beyond that distance. When the animated object drops to 0 hit points, it reverts to its original object form, and any remaining damage carries over to its original object form.",
"If you command an object to attack, it can make a single melee attack against a creature within 5 feet of it. It makes a slam attack with an attack bonus and bludgeoning damage determined by its size. The DM might rule that a specific object inflicts slashing or piercing damage based on its form."
],
"higher_level": [
"If you cast this spell using a spell slot of 6th level or higher, you can animate two additional objects for each slot level above 5th."
],
"range": "120 feet",
"components": [
"V",
"S"
],
"ritual": false,
"duration": "Up to 1 minute",
"concentration": true,
"casting_time": "1 action",
"level": 5,
"school": {
"index": "transmutation",
"name": "Transmutation",
"url": "/api/magic-schools/transmutation"
},
"classes": [
{
"index": "bard",
"name": "Bard",
"url": "/api/classes/bard"
},
{
"index": "sorcerer",
"name": "Sorcerer",
"url": "/api/classes/sorcerer"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [],
"url": "/api/spells/animate-objects"
}
]

384
dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-3.snap Normal file
View File

@ -0,0 +1,384 @@
---
source: dump/src/reader/v4/mod.rs
expression: tasks
---
[
{
"id": 9,
"index_uid": "movies_2",
"content": {
"DocumentAddition": {
"content_uuid": "3b12a971-bca2-4716-9889-36ffb715ae1d",
"merge_strategy": "ReplaceDocuments",
"primary_key": null,
"documents_count": 200,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:49.125132233Z"
}
]
},
{
"id": 8,
"index_uid": "movies",
"content": {
"DocumentAddition": {
"content_uuid": "cae3205a-6016-471b-81de-081a195f098c",
"merge_strategy": "ReplaceDocuments",
"primary_key": null,
"documents_count": 100,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:49.114226973Z"
},
{
"Batched": {
"timestamp": "2022-10-06T12:53:49.125918825Z",
"batch_id": 8
}
},
{
"Processing": "2022-10-06T12:53:49.125930546Z"
},
{
"Succeded": {
"result": {
"DocumentAddition": {
"indexed_documents": 100
}
},
"timestamp": "2022-10-06T12:53:49.785862546Z"
}
}
]
},
{
"id": 7,
"index_uid": "dnd_spells",
"content": {
"DocumentAddition": {
"content_uuid": "7ba1eaa0-d2fb-4852-8d00-f35ed166728f",
"merge_strategy": "ReplaceDocuments",
"primary_key": "index",
"documents_count": 10,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:41.070732179Z"
},
{
"Batched": {
"timestamp": "2022-10-06T12:53:41.085563291Z",
"batch_id": 7
}
},
{
"Processing": "2022-10-06T12:53:41.085563961Z"
},
{
"Succeded": {
"result": {
"DocumentAddition": {
"indexed_documents": 10
}
},
"timestamp": "2022-10-06T12:53:41.116036186Z"
}
}
]
},
{
"id": 6,
"index_uid": "dnd_spells",
"content": {
"DocumentAddition": {
"content_uuid": "f2fb7d6e-11b6-45d9-aa7a-9495a567a275",
"merge_strategy": "ReplaceDocuments",
"primary_key": null,
"documents_count": 10,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:40.831649057Z"
},
{
"Batched": {
"timestamp": "2022-10-06T12:53:40.834515892Z",
"batch_id": 6
}
},
{
"Processing": "2022-10-06T12:53:40.834516572Z"
},
{
"Failed": {
"error": {
"message": "The primary key inference process failed because the engine did not find any fields containing `id` substring in their name. If your document identifier does not contain any `id` substring, you can set the primary key of the index.",
"code": "primary_key_inference_failed",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#primary_key_inference_failed"
},
"timestamp": "2022-10-06T12:53:40.905384918Z"
}
}
]
},
{
"id": 5,
"index_uid": "products",
"content": {
"DocumentAddition": {
"content_uuid": "f269fe46-36fe-4fe7-8c4e-2054f1b23594",
"merge_strategy": "ReplaceDocuments",
"primary_key": "sku",
"documents_count": 10,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:40.576727649Z"
},
{
"Batched": {
"timestamp": "2022-10-06T12:53:40.587595408Z",
"batch_id": 5
}
},
{
"Processing": "2022-10-06T12:53:40.587596158Z"
},
{
"Succeded": {
"result": {
"DocumentAddition": {
"indexed_documents": 10
}
},
"timestamp": "2022-10-06T12:53:40.603035979Z"
}
}
]
},
{
"id": 4,
"index_uid": "products",
"content": {
"DocumentAddition": {
"content_uuid": "7d1ea292-cdb6-4f47-8b25-c2ddde89035c",
"merge_strategy": "ReplaceDocuments",
"primary_key": null,
"documents_count": 10,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:39.979427178Z"
},
{
"Batched": {
"timestamp": "2022-10-06T12:53:39.986159313Z",
"batch_id": 4
}
},
{
"Processing": "2022-10-06T12:53:39.986160113Z"
},
{
"Failed": {
"error": {
"message": "The primary key inference process failed because the engine did not find any fields containing `id` substring in their name. If your document identifier does not contain any `id` substring, you can set the primary key of the index.",
"code": "primary_key_inference_failed",
"type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#primary_key_inference_failed"
},
"timestamp": "2022-10-06T12:53:39.98921592Z"
}
}
]
},
{
"id": 3,
"index_uid": "products",
"content": {
"SettingsUpdate": {
"settings": {
"synonyms": {
"android": [
"phone",
"smartphone"
],
"iphone": [
"phone",
"smartphone"
],
"phone": [
"smartphone",
"iphone",
"android"
]
}
},
"is_deletion": false,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:39.360187055Z"
},
{
"Batched": {
"timestamp": "2022-10-06T12:53:39.371250258Z",
"batch_id": 3
}
},
{
"Processing": "2022-10-06T12:53:39.371250918Z"
},
{
"Processing": "2022-10-06T12:53:39.373988491Z"
},
{
"Succeded": {
"result": "Other",
"timestamp": "2022-10-06T12:53:39.449840865Z"
}
}
]
},
{
"id": 2,
"index_uid": "movies",
"content": {
"SettingsUpdate": {
"settings": {
"rankingRules": [
"words",
"typo",
"proximity",
"attribute",
"sort",
"exactness",
"release_date:asc"
]
},
"is_deletion": false,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:39.143829637Z"
},
{
"Batched": {
"timestamp": "2022-10-06T12:53:39.154803808Z",
"batch_id": 2
}
},
{
"Processing": "2022-10-06T12:53:39.154804558Z"
},
{
"Processing": "2022-10-06T12:53:39.157501241Z"
},
{
"Succeded": {
"result": "Other",
"timestamp": "2022-10-06T12:53:39.160263154Z"
}
}
]
},
{
"id": 1,
"index_uid": "movies",
"content": {
"SettingsUpdate": {
"settings": {
"filterableAttributes": [
"genres",
"id"
],
"sortableAttributes": [
"release_date"
]
},
"is_deletion": false,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:38.922837679Z"
},
{
"Batched": {
"timestamp": "2022-10-06T12:53:38.937712641Z",
"batch_id": 1
}
},
{
"Processing": "2022-10-06T12:53:38.937713141Z"
},
{
"Processing": "2022-10-06T12:53:38.940482335Z"
},
{
"Succeded": {
"result": "Other",
"timestamp": "2022-10-06T12:53:38.953566059Z"
}
}
]
},
{
"id": 0,
"index_uid": "movies",
"content": {
"DocumentAddition": {
"content_uuid": "cee1eef7-fadd-4970-93dc-25518655175f",
"merge_strategy": "ReplaceDocuments",
"primary_key": null,
"documents_count": 10,
"allow_index_creation": true
}
},
"events": [
{
"Created": "2022-10-06T12:53:38.710611568Z"
},
{
"Batched": {
"timestamp": "2022-10-06T12:53:38.717455314Z",
"batch_id": 0
}
},
{
"Processing": "2022-10-06T12:53:38.717456194Z"
},
{
"Succeded": {
"result": {
"DocumentAddition": {
"indexed_documents": 10
}
},
"timestamp": "2022-10-06T12:53:38.811687295Z"
}
}
]
}
]

50
dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-4.snap Normal file
View File

@ -0,0 +1,50 @@
---
source: dump/src/reader/v4/mod.rs
expression: keys
---
[
{
"description": "Default Search API Key (Use it to search from the frontend)",
"id": [
110,
113,
57,
52,
113,
97,
71,
106
],
"actions": [
"search"
],
"indexes": [
"*"
],
"expires_at": null,
"created_at": "2022-10-06T12:53:33.424274047Z",
"updated_at": "2022-10-06T12:53:33.424274047Z"
},
{
"description": "Default Admin API Key (Use it for all other operations. Caution! Do not use it on a public frontend)",
"id": [
105,
121,
109,
83,
109,
111,
53,
83
],
"actions": [
"*"
],
"indexes": [
"*"
],
"expires_at": null,
"created_at": "2022-10-06T12:53:33.417707446Z",
"updated_at": "2022-10-06T12:53:33.417707446Z"
}
]

71
dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-6.snap Normal file
View File

@ -0,0 +1,71 @@
---
source: dump/src/reader/v4/mod.rs
expression: products.settings()
---
Ok(
Settings {
displayed_attributes: Reset,
searchable_attributes: Reset,
filterable_attributes: Set(
{},
),
sortable_attributes: Set(
{},
),
ranking_rules: Set(
[
"words",
"typo",
"proximity",
"attribute",
"sort",
"exactness",
],
),
stop_words: Set(
{},
),
synonyms: Set(
{
"android": [
"phone",
"smartphone",
],
"iphone": [
"phone",
"smartphone",
],
"phone": [
"android",
"iphone",
"smartphone",
],
},
),
distinct_attribute: Reset,
typo_tolerance: Set(
TypoSettings {
enabled: Set(
true,
),
min_word_size_for_typos: Set(
MinWordSizeTyposSetting {
one_typo: Set(
5,
),
two_typos: Set(
9,
),
},
),
disable_on_words: Set(
{},
),
disable_on_attributes: Set(
{},
),
},
),
_kind: PhantomData<dump::reader::v4::settings::Checked>,
},
)

308
dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-7.snap Normal file
View File

@ -0,0 +1,308 @@
---
source: dump/src/reader/v4/mod.rs
expression: documents
---
[
{
"sku": 43900,
"name": "Duracell - AAA Batteries (4-Pack)",
"type": "HardGood",
"price": 5.49,
"upc": "041333424019",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Compatible with select electronic devices; AAA size; DURALOCK Power Preserve technology; 4-pack",
"manufacturer": "Duracell",
"model": "MN2400B4Z",
"url": "http://www.bestbuy.com/site/duracell-aaa-batteries-4-pack/43900.p?id=1051384074145&skuId=43900&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/4390/43900_sa.jpg"
},
{
"sku": 48530,
"name": "Duracell - AA 1.5V CopperTop Batteries (4-Pack)",
"type": "HardGood",
"price": 5.49,
"upc": "041333415017",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Long-lasting energy; DURALOCK Power Preserve technology; for toys, clocks, radios, games, remotes, PDAs and more",
"manufacturer": "Duracell",
"model": "MN1500B4Z",
"url": "http://www.bestbuy.com/site/duracell-aa-1-5v-coppertop-batteries-4-pack/48530.p?id=1099385268988&skuId=48530&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/4853/48530_sa.jpg"
},
{
"sku": 127687,
"name": "Duracell - AA Batteries (8-Pack)",
"type": "HardGood",
"price": 7.49,
"upc": "041333825014",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Compatible with select electronic devices; AA size; DURALOCK Power Preserve technology; 8-pack",
"manufacturer": "Duracell",
"model": "MN1500B8Z",
"url": "http://www.bestbuy.com/site/duracell-aa-batteries-8-pack/127687.p?id=1051384045676&skuId=127687&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/1276/127687_sa.jpg"
},
{
"sku": 150115,
"name": "Energizer - MAX Batteries AA (4-Pack)",
"type": "HardGood",
"price": 4.99,
"upc": "039800011329",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "4-pack AA alkaline batteries; battery tester included",
"manufacturer": "Energizer",
"model": "E91BP-4",
"url": "http://www.bestbuy.com/site/energizer-max-batteries-aa-4-pack/150115.p?id=1051384046217&skuId=150115&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/1501/150115_sa.jpg"
},
{
"sku": 185230,
"name": "Duracell - C Batteries (4-Pack)",
"type": "HardGood",
"price": 8.99,
"upc": "041333440019",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Compatible with select electronic devices; C size; DURALOCK Power Preserve technology; 4-pack",
"manufacturer": "Duracell",
"model": "MN1400R4Z",
"url": "http://www.bestbuy.com/site/duracell-c-batteries-4-pack/185230.p?id=1051384046486&skuId=185230&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/1852/185230_sa.jpg"
},
{
"sku": 185267,
"name": "Duracell - D Batteries (4-Pack)",
"type": "HardGood",
"price": 9.99,
"upc": "041333430010",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.99,
"description": "Compatible with select electronic devices; D size; DURALOCK Power Preserve technology; 4-pack",
"manufacturer": "Duracell",
"model": "MN1300R4Z",
"url": "http://www.bestbuy.com/site/duracell-d-batteries-4-pack/185267.p?id=1051384046551&skuId=185267&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/1852/185267_sa.jpg"
},
{
"sku": 312290,
"name": "Duracell - 9V Batteries (2-Pack)",
"type": "HardGood",
"price": 7.99,
"upc": "041333216010",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Compatible with select electronic devices; alkaline chemistry; 9V size; DURALOCK Power Preserve technology; 2-pack",
"manufacturer": "Duracell",
"model": "MN1604B2Z",
"url": "http://www.bestbuy.com/site/duracell-9v-batteries-2-pack/312290.p?id=1051384050321&skuId=312290&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/3122/312290_sa.jpg"
},
{
"sku": 324884,
"name": "Directed Electronics - Viper Audio Glass Break Sensor",
"type": "HardGood",
"price": 39.99,
"upc": "093207005060",
"category": [
{
"id": "pcmcat113100050015",
"name": "Carfi Instore Only"
}
],
"shipping": 0,
"description": "From our expanded online assortment; compatible with Directed Electronics alarm systems; microphone and microprocessor detect and analyze intrusions; detects quiet glass breaks",
"manufacturer": "Directed Electronics",
"model": "506T",
"url": "http://www.bestbuy.com/site/directed-electronics-viper-audio-glass-break-sensor/324884.p?id=1112808077651&skuId=324884&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/3248/324884_rc.jpg"
},
{
"sku": 333179,
"name": "Energizer - N Cell E90 Batteries (2-Pack)",
"type": "HardGood",
"price": 5.99,
"upc": "039800013200",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208006",
"name": "Specialty Batteries"
}
],
"shipping": 5.49,
"description": "Alkaline batteries; 1.5V",
"manufacturer": "Energizer",
"model": "E90BP-2",
"url": "http://www.bestbuy.com/site/energizer-n-cell-e90-batteries-2-pack/333179.p?id=1185268509951&skuId=333179&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/3331/333179_sa.jpg"
},
{
"sku": 346575,
"name": "Metra - Radio Installation Dash Kit for Most 1989-2000 Ford, Lincoln & Mercury Vehicles - Black",
"type": "HardGood",
"price": 16.99,
"upc": "086429002757",
"category": [
{
"id": "abcat0300000",
"name": "Car Electronics & GPS"
},
{
"id": "pcmcat165900050023",
"name": "Car Installation Parts & Accessories"
},
{
"id": "pcmcat331600050007",
"name": "Car Audio Installation Parts"
},
{
"id": "pcmcat165900050031",
"name": "Deck Installation Parts"
},
{
"id": "pcmcat165900050033",
"name": "Dash Installation Kits"
}
],
"shipping": 0,
"description": "From our expanded online assortment; compatible with most 1989-2000 Ford, Lincoln and Mercury vehicles; snap-in TurboKit offers fast installation; spacer/trim ring; rear support bracket",
"manufacturer": "Metra",
"model": "99-5512",
"url": "http://www.bestbuy.com/site/metra-radio-installation-dash-kit-for-most-1989-2000-ford-lincoln-mercury-vehicles-black/346575.p?id=1218118704590&skuId=346575&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/3465/346575_rc.jpg"
}
]

63
dump/src/reader/v4/snapshots/dump__reader__v4__test__read_dump_v4-9.snap Normal file
View File

@ -0,0 +1,63 @@
---
source: dump/src/reader/v4/mod.rs
expression: movies.settings()
---
Ok(
Settings {
displayed_attributes: Reset,
searchable_attributes: Reset,
filterable_attributes: Set(
{
"genres",
"id",
},
),
sortable_attributes: Set(
{
"release_date",
},
),
ranking_rules: Set(
[
"words",
"typo",
"proximity",
"attribute",
"sort",
"exactness",
"release_date:asc",
],
),
stop_words: Set(
{},
),
synonyms: Set(
{},
),
distinct_attribute: Reset,
typo_tolerance: Set(
TypoSettings {
enabled: Set(
true,
),
min_word_size_for_typos: Set(
MinWordSizeTyposSetting {
one_typo: Set(
5,
),
two_typos: Set(
9,
),
},
),
disable_on_words: Set(
{},
),
disable_on_attributes: Set(
{},
),
},
),
_kind: PhantomData<dump::reader::v4::settings::Checked>,
},
)

454
dump/src/reader/v4/tasks.rs Normal file
View File

@ -0,0 +1,454 @@
use std::fmt::Write;
use serde::{Deserialize, Serializer};
use time::{Duration, OffsetDateTime};
use uuid::Uuid;
use super::{
meta::IndexUid,
settings::{Settings, Unchecked},
};
pub type TaskId = u32;
pub type BatchId = u32;
#[derive(Clone, Debug, Deserialize, PartialEq)]
#[cfg_attr(test, derive(serde::Serialize))]
pub struct Task {
pub id: TaskId,
pub index_uid: IndexUid,
pub content: TaskContent,
pub events: Vec<TaskEvent>,
}
#[derive(Clone, Debug, Deserialize, PartialEq)]
#[cfg_attr(test, derive(serde::Serialize))]
#[allow(clippy::large_enum_variant)]
pub enum TaskContent {
DocumentAddition {
content_uuid: Uuid,
merge_strategy: IndexDocumentsMethod,
primary_key: Option<String>,
documents_count: usize,
allow_index_creation: bool,
},
DocumentDeletion(DocumentDeletion),
SettingsUpdate {
settings: Settings<Unchecked>,
/// Indicates whether the task was a deletion
is_deletion: bool,
allow_index_creation: bool,
},
IndexDeletion,
IndexCreation {
primary_key: Option<String>,
},
IndexUpdate {
primary_key: Option<String>,
},
}
#[derive(Debug)]
#[cfg_attr(test, derive(serde::Serialize))]
#[cfg_attr(test, serde(untagged))]
#[allow(clippy::large_enum_variant)]
enum TaskDetails {
#[cfg_attr(test, serde(rename_all = "camelCase"))]
DocumentAddition {
received_documents: usize,
indexed_documents: Option<u64>,
},
#[cfg_attr(test, serde(rename_all = "camelCase"))]
Settings {
#[cfg_attr(test, serde(flatten))]
settings: Settings<Unchecked>,
},
#[cfg_attr(test, serde(rename_all = "camelCase"))]
IndexInfo { primary_key: Option<String> },
#[cfg_attr(test, serde(rename_all = "camelCase"))]
DocumentDeletion {
received_document_ids: usize,
deleted_documents: Option<u64>,
},
#[cfg_attr(test, serde(rename_all = "camelCase"))]
ClearAll { deleted_documents: Option<u64> },
}
#[derive(Debug)]
#[cfg_attr(test, derive(serde::Serialize))]
#[cfg_attr(test, serde(rename_all = "camelCase"))]
enum TaskStatus {
Enqueued,
Processing,
Succeeded,
Failed,
}
#[derive(Debug)]
#[cfg_attr(test, derive(serde::Serialize))]
#[cfg_attr(test, serde(rename_all = "camelCase"))]
enum TaskType {
IndexCreation,
IndexUpdate,
IndexDeletion,
DocumentAddition,
DocumentPartial,
DocumentDeletion,
SettingsUpdate,
ClearAll,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize)]
#[cfg_attr(test, derive(serde::Serialize))]
pub enum IndexDocumentsMethod {
/// Replace the previous document with the new one,
/// removing all the already known attributes.
ReplaceDocuments,
/// Merge the previous version of the document with the new version,
/// replacing old attributes values with the new ones and add the new attributes.
UpdateDocuments,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
#[cfg_attr(test, derive(serde::Serialize))]
pub enum DocumentDeletion {
Clear,
Ids(Vec<String>),
}
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
#[cfg_attr(test, derive(serde::Serialize))]
pub enum TaskEvent {
Created(#[serde(with = "time::serde::rfc3339")] OffsetDateTime),
Batched {
#[serde(with = "time::serde::rfc3339")]
timestamp: OffsetDateTime,
batch_id: BatchId,
},
Processing(#[serde(with = "time::serde::rfc3339")] OffsetDateTime),
Succeded {
result: TaskResult,
#[serde(with = "time::serde::rfc3339")]
timestamp: OffsetDateTime,
},
Failed {
error: ResponseError,
#[serde(with = "time::serde::rfc3339")]
timestamp: OffsetDateTime,
},
}
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
#[cfg_attr(test, derive(serde::Serialize))]
pub enum TaskResult {
DocumentAddition { indexed_documents: u64 },
DocumentDeletion { deleted_documents: u64 },
ClearAll { deleted_documents: u64 },
Other,
}
#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
#[cfg_attr(test, derive(serde::Serialize))]
#[serde(rename_all = "camelCase")]
pub struct ResponseError {
pub message: String,
#[serde(rename = "code")]
pub error_code: String,
#[serde(rename = "type")]
pub error_type: String,
#[serde(rename = "link")]
pub error_link: String,
}
impl Task {
/// Return true when a task is finished.
/// A task is finished when its last state is either `Succeeded` or `Failed`.
pub fn is_finished(&self) -> bool {
self.events.last().map_or(false, |event| {
matches!(event, TaskEvent::Succeded { .. } | TaskEvent::Failed { .. })
})
}
/// Return the content_uuid of the `Task` if there is one.
pub fn get_content_uuid(&self) -> Option<Uuid> {
match self {
Task {
content: TaskContent::DocumentAddition { content_uuid, .. },
..
} => Some(*content_uuid),
_ => None,
}
}
}
impl IndexUid {
pub fn into_inner(self) -> String {
self.0
}
/// Return a reference over the inner str.
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::ops::Deref for IndexUid {
type Target = str;
fn deref(&self) -> &Self::Target {
&self.0
}
}
#[derive(Debug)]
#[cfg_attr(test, derive(serde::Serialize))]
#[cfg_attr(test, serde(rename_all = "camelCase"))]
pub struct TaskView {
uid: TaskId,
index_uid: String,
status: TaskStatus,
#[cfg_attr(test, serde(rename = "type"))]
task_type: TaskType,
#[cfg_attr(test, serde(skip_serializing_if = "Option::is_none"))]
details: Option<TaskDetails>,
#[cfg_attr(test, serde(skip_serializing_if = "Option::is_none"))]
error: Option<ResponseError>,
#[cfg_attr(test, serde(serialize_with = "serialize_duration"))]
duration: Option<Duration>,
#[cfg_attr(test, serde(serialize_with = "time::serde::rfc3339::serialize"))]
enqueued_at: OffsetDateTime,
#[cfg_attr(
test,
serde(serialize_with = "time::serde::rfc3339::option::serialize")
)]
started_at: Option<OffsetDateTime>,
#[cfg_attr(
test,
serde(serialize_with = "time::serde::rfc3339::option::serialize")
)]
finished_at: Option<OffsetDateTime>,
#[cfg_attr(test, serde(skip_serializing_if = "Option::is_none"))]
batch_uid: Option<Option<BatchId>>,
}
impl From<Task> for TaskView {
fn from(task: Task) -> Self {
let Task {
id,
index_uid,
content,
events,
} = task;
let (task_type, mut details) = match content {
TaskContent::DocumentAddition {
merge_strategy,
documents_count,
..
} => {
let details = TaskDetails::DocumentAddition {
received_documents: documents_count,
indexed_documents: None,
};
let task_type = match merge_strategy {
IndexDocumentsMethod::UpdateDocuments => TaskType::DocumentPartial,
IndexDocumentsMethod::ReplaceDocuments => TaskType::DocumentAddition,
_ => unreachable!("Unexpected document merge strategy."),
};
(task_type, Some(details))
}
TaskContent::DocumentDeletion(DocumentDeletion::Ids(ids)) => (
TaskType::DocumentDeletion,
Some(TaskDetails::DocumentDeletion {
received_document_ids: ids.len(),
deleted_documents: None,
}),
),
TaskContent::DocumentDeletion(DocumentDeletion::Clear) => (
TaskType::ClearAll,
Some(TaskDetails::ClearAll {
deleted_documents: None,
}),
),
TaskContent::IndexDeletion => (
TaskType::IndexDeletion,
Some(TaskDetails::ClearAll {
deleted_documents: None,
}),
),
TaskContent::SettingsUpdate { settings, .. } => (
TaskType::SettingsUpdate,
Some(TaskDetails::Settings { settings }),
),
TaskContent::IndexCreation { primary_key } => (
TaskType::IndexCreation,
Some(TaskDetails::IndexInfo { primary_key }),
),
TaskContent::IndexUpdate { primary_key } => (
TaskType::IndexUpdate,
Some(TaskDetails::IndexInfo { primary_key }),
),
};
// An event always has at least one event: "Created"
let (status, error, finished_at) = match events.last().unwrap() {
TaskEvent::Created(_) => (TaskStatus::Enqueued, None, None),
TaskEvent::Batched { .. } => (TaskStatus::Enqueued, None, None),
TaskEvent::Processing(_) => (TaskStatus::Processing, None, None),
TaskEvent::Succeded { timestamp, result } => {
match (result, &mut details) {
(
TaskResult::DocumentAddition {
indexed_documents: num,
..
},
Some(TaskDetails::DocumentAddition {
ref mut indexed_documents,
..
}),
) => {
indexed_documents.replace(*num);
}
(
TaskResult::DocumentDeletion {
deleted_documents: docs,
..
},
Some(TaskDetails::DocumentDeletion {
ref mut deleted_documents,
..
}),
) => {
deleted_documents.replace(*docs);
}
(
TaskResult::ClearAll {
deleted_documents: docs,
},
Some(TaskDetails::ClearAll {
ref mut deleted_documents,
}),
) => {
deleted_documents.replace(*docs);
}
_ => (),
}
(TaskStatus::Succeeded, None, Some(*timestamp))
}
TaskEvent::Failed { timestamp, error } => {
match details {
Some(TaskDetails::DocumentDeletion {
ref mut deleted_documents,
..
}) => {
deleted_documents.replace(0);
}
Some(TaskDetails::ClearAll {
ref mut deleted_documents,
..
}) => {
deleted_documents.replace(0);
}
Some(TaskDetails::DocumentAddition {
ref mut indexed_documents,
..
}) => {
indexed_documents.replace(0);
}
_ => (),
}
(TaskStatus::Failed, Some(error.clone()), Some(*timestamp))
}
};
let enqueued_at = match events.first() {
Some(TaskEvent::Created(ts)) => *ts,
_ => unreachable!("A task must always have a creation event."),
};
let started_at = events.iter().find_map(|e| match e {
TaskEvent::Processing(ts) => Some(*ts),
_ => None,
});
let duration = finished_at.zip(started_at).map(|(tf, ts)| (tf - ts));
let batch_uid = if true {
let id = events.iter().find_map(|e| match e {
TaskEvent::Batched { batch_id, .. } => Some(*batch_id),
_ => None,
});
Some(id)
} else {
None
};
Self {
uid: id,
index_uid: index_uid.into_inner(),
status,
task_type,
details,
error,
duration,
enqueued_at,
started_at,
finished_at,
batch_uid,
}
}
}
/// Serialize a `time::Duration` as a best effort ISO 8601 while waiting for
/// https://github.com/time-rs/time/issues/378.
/// This code is a port of the old code of time that was removed in 0.2.
fn serialize_duration<S: Serializer>(
duration: &Option<Duration>,
serializer: S,
) -> Result<S::Ok, S::Error> {
match duration {
Some(duration) => {
// technically speaking, negative duration is not valid ISO 8601
if duration.is_negative() {
return serializer.serialize_none();
}
const SECS_PER_DAY: i64 = Duration::DAY.whole_seconds();
let secs = duration.whole_seconds();
let days = secs / SECS_PER_DAY;
let secs = secs - days * SECS_PER_DAY;
let hasdate = days != 0;
let nanos = duration.subsec_nanoseconds();
let hastime = (secs != 0 || nanos != 0) || !hasdate;
// all the following unwrap can't fail
let mut res = String::new();
write!(&mut res, "P").unwrap();
if hasdate {
write!(&mut res, "{}D", days).unwrap();
}
const NANOS_PER_MILLI: i32 = Duration::MILLISECOND.subsec_nanoseconds();
const NANOS_PER_MICRO: i32 = Duration::MICROSECOND.subsec_nanoseconds();
if hastime {
if nanos == 0 {
write!(&mut res, "T{}S", secs).unwrap();
} else if nanos % NANOS_PER_MILLI == 0 {
write!(&mut res, "T{}.{:03}S", secs, nanos / NANOS_PER_MILLI).unwrap();
} else if nanos % NANOS_PER_MICRO == 0 {
write!(&mut res, "T{}.{:06}S", secs, nanos / NANOS_PER_MICRO).unwrap();
} else {
write!(&mut res, "T{}.{:09}S", secs, nanos).unwrap();
}
}
serializer.serialize_str(&res)
}
None => serializer.serialize_none(),
}
}

1
dump/src/reader/v5/mod.rs
View File

@ -141,7 +141,6 @@ impl V5Reader {
.join("updates")
.join("updates_files")
.join(uuid.to_string());
dbg!(&update_file_path);
Ok((task, Some(File::open(update_file_path).unwrap())))
} else {
Ok((task, None))

1
dump/src/writer.rs
View File

@ -304,7 +304,6 @@ pub(crate) mod test {
for (task, mut expected) in tasks_queue.lines().zip(create_test_tasks()) {
// TODO: This can be removed once `Duration` from the `TaskView` is implemented.
expected.0.duration = None;
dbg!(&task);
assert_eq!(serde_json::from_str::<TaskView>(task).unwrap(), expected.0);
if let Some(expected_update) = expected.1 {

BIN
dump/tests/assets/v4.dump Normal file
View File

Binary file not shown.