From 8d405fad12c9354d8281c683feb70cd557469462 Mon Sep 17 00:00:00 2001 From: curquiza Date: Wed, 30 Nov 2022 13:53:12 +0100 Subject: [PATCH 01/17] Simplify download-latest.sh script --- download-latest.sh | 120 ++++----------------------------------------- 1 file changed, 9 insertions(+), 111 deletions(-) diff --git a/download-latest.sh b/download-latest.sh index 42863d587..38e37e69e 100644 --- a/download-latest.sh +++ b/download-latest.sh @@ -10,9 +10,6 @@ DEFAULT='\033[0m' # Project name PNAME='meilisearch' -# Version regexp i.e. v[number].[number].[number] -GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' - # GitHub API address GITHUB_API='https://api.github.com/repos/meilisearch/meilisearch/releases' # GitHub Release address @@ -20,126 +17,26 @@ GITHUB_REL='https://github.com/meilisearch/meilisearch/releases/download/' # FUNCTIONS -# semverParseInto and semverLT from: https://github.com/cloudflare/semver_bash/blob/master/semver.sh -# usage: semverParseInto version major minor patch special -# version: the string version -# major, minor, patch, special: will be assigned by the function -semverParseInto() { - local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)' - # MAJOR - eval $2=`echo $1 | sed -e "s#$RE#\1#"` - # MINOR - eval $3=`echo $1 | sed -e "s#$RE#\2#"` - # PATCH - eval $4=`echo $1 | sed -e "s#$RE#\3#"` - # SPECIAL - eval $5=`echo $1 | sed -e "s#$RE#\4#"` -} - -# usage: semverLT version1 version2 -semverLT() { - local MAJOR_A=0 - local MINOR_A=0 - local PATCH_A=0 - local SPECIAL_A=0 - - local MAJOR_B=0 - local MINOR_B=0 - local PATCH_B=0 - local SPECIAL_B=0 - - semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A - semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B - - if [ $MAJOR_A -lt $MAJOR_B ]; then - return 0 - fi - if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then - return 0 - fi - if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then - return 0 - fi - if [ "_$SPECIAL_A" == '_' ] && [ "_$SPECIAL_B" == '_' ] ; then - return 1 - fi - if [ "_$SPECIAL_A" == '_' ] && [ "_$SPECIAL_B" != '_' ] ; then - return 1 - fi - if [ "_$SPECIAL_A" != '_' ] && [ "_$SPECIAL_B" == '_' ] ; then - return 0 - fi - if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then - return 0 - fi - - return 1 -} - -# Get a token from: https://github.com/settings/tokens to increase rate limit (from 60 to 5000), -# make sure the token scope is set to 'public_repo'. # Create GITHUB_PAT environment variable once you acquired the token to start using it. # Returns the tag of the latest stable release (in terms of semver and not of release date). get_latest() { # temp_file is needed because the grep would start before the download is over temp_file=$(mktemp -q /tmp/$PNAME.XXXXXXXXX) + latest_release="$GITHUB_API/latest" + if [ $? -ne 0 ]; then - echo "$0: Can't create temp file, bye bye.." + echo "$0: Can't create temp file." + fetch_release_failure_usage exit 1 fi if [ -z "$GITHUB_PAT" ]; then - curl -s $GITHUB_API > "$temp_file" || return 1 + curl -s "$latest_release" > "$temp_file" || return 1 else - curl -H "Authorization: token $GITHUB_PAT" -s $GITHUB_API > "$temp_file" || return 1 + curl -H "Authorization: token $GITHUB_PAT" -s "$latest_release" > "$temp_file" || return 1 fi - releases=$(cat "$temp_file" | \ - grep -E '"tag_name":|"draft":|"prerelease":' \ - | tr -d ',"' | cut -d ':' -f2 | tr -d ' ') - # Returns a list of [tag_name draft_boolean prerelease_boolean ...] - # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false... - - i=0 - latest='' - current_tag='' - for release_info in $releases; do - # Checking tag_name - if [ $i -eq 0 ]; then - # If it's not an alpha or beta release - if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then - current_tag=$release_info - else - current_tag='' - fi - i=1 - # Checking draft boolean - elif [ $i -eq 1 ]; then - if [ "$release_info" = 'true' ]; then - current_tag='' - fi - i=2 - # Checking prerelease boolean - elif [ $i -eq 2 ]; then - if [ "$release_info" = 'true' ]; then - current_tag='' - fi - i=0 - # If the current_tag is valid - if [ "$current_tag" != '' ]; then - # If there is no latest yes - if [ "$latest" = '' ]; then - latest="$current_tag" - else - # Comparing latest and the current tag - semverLT $current_tag $latest - if [ $? -eq 1 ]; then - latest="$current_tag" - fi - fi - fi - fi - done + latest="$(cat "$temp_file" | grep '"tag_name":' | cut -d ':' -f2 | tr -d '"' | tr -d ',' | tr -d ' ')" rm -f "$temp_file" return 0 @@ -210,12 +107,13 @@ fetch_release_failure_usage() { echo '' printf "$RED%s\n$DEFAULT" 'ERROR: Impossible to get the latest stable version of Meilisearch.' echo 'Please let us know about this issue: https://github.com/meilisearch/meilisearch/issues/new/choose' + echo '' + echo 'In the meantime, you can manually download the appropriate binary from the GitHub release assets here: https://github.com/meilisearch/meilisearch/releases/latest' } fill_release_variables() { # Fill $latest variable. if ! get_latest; then - # TO CHANGE. fetch_release_failure_usage exit 1 fi From eab1156f8cee2d417ca5242578bc10ca387938ad Mon Sep 17 00:00:00 2001 From: curquiza Date: Wed, 30 Nov 2022 14:14:46 +0100 Subject: [PATCH 02/17] Update script to be used with macOS apple silicon --- download-latest.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/download-latest.sh b/download-latest.sh index 38e37e69e..dca39fa44 100644 --- a/download-latest.sh +++ b/download-latest.sh @@ -1,5 +1,8 @@ #!/bin/sh +# This script works with a GitHub token to increase your request limit (for example, if using this script in a CI). +# To make it work, fill the GITHUB_PAT environment variable with your GitHub token. + # GLOBALS # Colors @@ -17,8 +20,8 @@ GITHUB_REL='https://github.com/meilisearch/meilisearch/releases/download/' # FUNCTIONS -# Create GITHUB_PAT environment variable once you acquired the token to start using it. -# Returns the tag of the latest stable release (in terms of semver and not of release date). +# Gets the version of the latest stable version of Meilisearch by setting the $latest variable. +# Returns 0 in case of success, 1 otherwise. get_latest() { # temp_file is needed because the grep would start before the download is over temp_file=$(mktemp -q /tmp/$PNAME.XXXXXXXXX) @@ -71,9 +74,9 @@ get_archi() { archi='amd64' ;; 'arm64') - # MacOS M1 + # macOS M1/M2 if [ $os = 'macos' ]; then - archi='amd64' + archi='apple-silicon' else archi='aarch64' fi From 9e3b1eb7a8bffa022743e55e7de576722af907e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar=20-=20curqui?= Date: Wed, 30 Nov 2022 16:55:27 +0100 Subject: [PATCH 03/17] Update download-latest.sh Co-authored-by: Louis Dureuil --- download-latest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download-latest.sh b/download-latest.sh index dca39fa44..81651a9a8 100644 --- a/download-latest.sh +++ b/download-latest.sh @@ -1,6 +1,6 @@ #!/bin/sh -# This script works with a GitHub token to increase your request limit (for example, if using this script in a CI). +# This script can optionally use a GitHub token to increase your request limit (for example, if using this script in a CI). # To make it work, fill the GITHUB_PAT environment variable with your GitHub token. # GLOBALS From da8044f91ef13d276676d06a80b9b4418a22e1a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar=20-=20curqui?= Date: Wed, 30 Nov 2022 16:55:32 +0100 Subject: [PATCH 04/17] Update download-latest.sh Co-authored-by: Louis Dureuil --- download-latest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download-latest.sh b/download-latest.sh index 81651a9a8..aa0b6e4d7 100644 --- a/download-latest.sh +++ b/download-latest.sh @@ -1,7 +1,7 @@ #!/bin/sh # This script can optionally use a GitHub token to increase your request limit (for example, if using this script in a CI). -# To make it work, fill the GITHUB_PAT environment variable with your GitHub token. +# To use a GitHub token, pass it through the GITHUB_PAT environment variable. # GLOBALS From 5d22c7bccef8255e7ca483b75bfc4f746226dea3 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 30 Nov 2022 14:51:21 +0100 Subject: [PATCH 05/17] Make some dump types Clone --- dump/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dump/src/lib.rs b/dump/src/lib.rs index 5be680c12..7a7b9a5b7 100644 --- a/dump/src/lib.rs +++ b/dump/src/lib.rs @@ -23,7 +23,7 @@ const CURRENT_DUMP_VERSION: Version = Version::V6; type Result = std::result::Result; -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Metadata { pub dump_version: Version, @@ -32,7 +32,7 @@ pub struct Metadata { pub dump_date: OffsetDateTime, } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { pub uid: String, @@ -43,7 +43,7 @@ pub struct IndexMetadata { pub updated_at: OffsetDateTime, } -#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] pub enum Version { V1, V2, From d44652209dce455cf45513330836dd8e88b585af Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 30 Nov 2022 14:52:20 +0100 Subject: [PATCH 06/17] impl Display for v2::settings::Criterion --- dump/src/reader/v2/settings.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dump/src/reader/v2/settings.rs b/dump/src/reader/v2/settings.rs index 62e5c05f9..1a7935b56 100644 --- a/dump/src/reader/v2/settings.rs +++ b/dump/src/reader/v2/settings.rs @@ -1,4 +1,5 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Display; use std::marker::PhantomData; use std::str::FromStr; @@ -174,3 +175,17 @@ impl FromStr for Criterion { } } } + +impl Display for Criterion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Criterion::Words => write!(f, "words"), + Criterion::Typo => write!(f, "typo"), + Criterion::Proximity => write!(f, "proximity"), + Criterion::Attribute => write!(f, "attribute"), + Criterion::Exactness => write!(f, "exactness"), + Criterion::Asc(field_name) => write!(f, "asc({})", field_name), + Criterion::Desc(field_name) => write!(f, "desc({})", field_name), + } + } +} From b9a8533de126398af0d2bb52ebee36ecf8285448 Mon Sep 17 00:00:00 2001 From: curquiza Date: Thu, 1 Dec 2022 12:48:01 +0100 Subject: [PATCH 07/17] Add CI to push a latest git tag for every stable Meilisearch release --- .github/workflows/latest-git-tag.yml | 19 +++++++++++++++++++ .github/workflows/publish-docker-images.yml | 8 ++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/latest-git-tag.yml diff --git a/.github/workflows/latest-git-tag.yml b/.github/workflows/latest-git-tag.yml new file mode 100644 index 000000000..9d9339ca8 --- /dev/null +++ b/.github/workflows/latest-git-tag.yml @@ -0,0 +1,19 @@ +# Create or update a latest git tag when releasing a stable vesrsin of Meilisearch +name: Update latest git tag +on: + workflow_dispatch: + release: + types: [released] + +jobs: + update-latest-tag: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: rickstaa/action-create-tag@v1 + with: + tag: "latest" + message: "Latest stable release of Meilisearch" + # Move the tag if `latest` already exists + force_push_tag: true + github_token: ${{ secrets.MEILI_BOT_GH_PAT }} diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index 3abe9ce47..a168c0bd8 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -3,8 +3,12 @@ on: schedule: - cron: '0 4 * * *' # Every day at 4:00am push: - tags: - - '*' + # Will run for every tag pushed except `latest` + # When the `latest` git tag is created with this [CI](../latest-git-tag.yml) + # we don't need to create a Docker `latest` image again. + # The `latest` Docker image push is already done in this CI when releasing a stable version of Meilisearch. + tags-ignore: + - latest name: Publish tagged images to Docker Hub From b8de369e3344d0947e9b8cff470fd376625b3dd3 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 30 Nov 2022 14:54:34 +0100 Subject: [PATCH 08/17] Add v1 reader --- dump/src/reader/v1/mod.rs | 318 +++++++++++------- ...mp__reader__v1__test__read_dump_v1-10.snap | 24 ++ ...mp__reader__v1__test__read_dump_v1-12.snap | 24 ++ ...ump__reader__v1__test__read_dump_v1-2.snap | 38 +++ ...ump__reader__v1__test__read_dump_v1-5.snap | 28 ++ ...ump__reader__v1__test__read_dump_v1-6.snap | 28 ++ ...ump__reader__v1__test__read_dump_v1-7.snap | 28 ++ ...ump__reader__v1__test__read_dump_v1-8.snap | 24 ++ dump/src/reader/v1/update.rs | 46 --- dump/src/reader/v1/v1.rs | 22 -- dump/tests/assets/v1.dump | Bin 0 -> 9904 bytes 11 files changed, 398 insertions(+), 182 deletions(-) create mode 100644 dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-10.snap create mode 100644 dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-12.snap create mode 100644 dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-2.snap create mode 100644 dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-5.snap create mode 100644 dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-6.snap create mode 100644 dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-7.snap create mode 100644 dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-8.snap delete mode 100644 dump/src/reader/v1/v1.rs create mode 100644 dump/tests/assets/v1.dump diff --git a/dump/src/reader/v1/mod.rs b/dump/src/reader/v1/mod.rs index f638262cc..1932b602a 100644 --- a/dump/src/reader/v1/mod.rs +++ b/dump/src/reader/v1/mod.rs @@ -1,173 +1,263 @@ use std::{ - convert::Infallible, fs::{self, File}, io::{BufRead, BufReader}, - path::Path, + path::{Path, PathBuf}, }; use tempfile::TempDir; use time::OffsetDateTime; -use self::update::UpdateStatus; - -use super::{DumpReader, IndexReader}; -use crate::{Error, Result, Version}; +use super::{compat::v1_to_v2::CompatV1ToV2, Document}; +use crate::{IndexMetadata, Result, Version}; +use serde::Deserialize; pub mod settings; pub mod update; -pub mod v1; pub struct V1Reader { - dump: TempDir, - metadata: v1::Metadata, - indexes: Vec, + pub dump: TempDir, + pub db_version: String, + pub dump_version: crate::Version, + indexes: Vec, } -struct V1IndexReader { - name: String, +pub struct IndexUuid { + pub name: String, + pub uid: String, +} +pub type Task = self::update::UpdateStatus; + +struct V1Index { + metadata: IndexMetadataV1, + path: PathBuf, +} + +impl V1Index { + pub fn new(path: PathBuf, metadata: Index) -> Self { + Self { metadata: metadata.into(), path } + } + + pub fn open(&self) -> Result { + V1IndexReader::new(&self.path, self.metadata.clone()) + } + + pub fn metadata(&self) -> &IndexMetadata { + &self.metadata.metadata + } +} + +pub struct V1IndexReader { + metadata: IndexMetadataV1, documents: BufReader, settings: BufReader, updates: BufReader, - - current_update: Option, } impl V1IndexReader { - pub fn new(name: String, path: &Path) -> Result { - let mut ret = V1IndexReader { - name, + pub fn new(path: &Path, metadata: IndexMetadataV1) -> Result { + Ok(V1IndexReader { + metadata, documents: BufReader::new(File::open(path.join("documents.jsonl"))?), settings: BufReader::new(File::open(path.join("settings.json"))?), updates: BufReader::new(File::open(path.join("updates.jsonl"))?), - current_update: None, - }; - ret.next_update(); - - Ok(ret) + }) } - pub fn next_update(&mut self) -> Result> { - let current_update = if let Some(line) = self.updates.lines().next() { - Some(serde_json::from_str(&line?)?) - } else { - None - }; + pub fn metadata(&self) -> &IndexMetadata { + &self.metadata.metadata + } - Ok(std::mem::replace(&mut self.current_update, current_update)) + pub fn documents(&mut self) -> Result> + '_> { + Ok((&mut self.documents) + .lines() + .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })) + } + + pub fn settings(&mut self) -> Result { + Ok(serde_json::from_reader(&mut self.settings)?) + } + + pub fn tasks(self) -> impl Iterator> { + self.updates.lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }) } } impl V1Reader { pub fn open(dump: TempDir) -> Result { - let mut meta_file = fs::read(dump.path().join("metadata.json"))?; - let metadata = serde_json::from_reader(&*meta_file)?; + let meta_file = fs::read(dump.path().join("metadata.json"))?; + let metadata: Metadata = serde_json::from_reader(&*meta_file)?; let mut indexes = Vec::new(); - let entries = fs::read_dir(dump.path())?; - for entry in entries { - let entry = entry?; - if entry.file_type()?.is_dir() { - indexes.push(V1IndexReader::new( - entry - .file_name() - .to_str() - .ok_or(Error::BadIndexName)? - .to_string(), - &entry.path(), - )?); - } + for index in metadata.indexes.into_iter() { + let index_path = dump.path().join(&index.uid); + indexes.push(V1Index::new(index_path, index)); } Ok(V1Reader { dump, - metadata, indexes, + db_version: metadata.db_version, + dump_version: metadata.dump_version, }) } - fn next_update(&mut self) -> Result> { - if let Some((idx, _)) = self - .indexes + pub fn to_v2(self) -> CompatV1ToV2 { + CompatV1ToV2 { from: self } + } + + pub fn index_uuid(&self) -> Vec { + self.indexes .iter() - .map(|index| index.current_update) - .enumerate() - .filter_map(|(idx, update)| update.map(|u| (idx, u))) - .min_by_key(|(_, update)| update.enqueued_at()) - { - self.indexes[idx].next_update() - } else { - Ok(None) + .map(|index| IndexUuid { + name: index.metadata.name.to_owned(), + uid: index.metadata().uid.to_owned(), + }) + .collect() + } + + pub fn version(&self) -> Version { + Version::V1 + } + + pub fn date(&self) -> Option { + None + } + + pub fn indexes(&self) -> Result> + '_> { + Ok(self.indexes.iter().map(|index| index.open())) + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Index { + pub name: String, + pub uid: String, + #[serde(with = "time::serde::rfc3339")] + created_at: OffsetDateTime, + #[serde(with = "time::serde::rfc3339")] + updated_at: OffsetDateTime, + pub primary_key: Option, +} + +#[derive(Clone)] +pub struct IndexMetadataV1 { + pub name: String, + pub metadata: crate::IndexMetadata, +} + +impl From for IndexMetadataV1 { + fn from(index: Index) -> Self { + IndexMetadataV1 { + name: index.name, + metadata: crate::IndexMetadata { + uid: index.uid, + primary_key: index.primary_key, + created_at: index.created_at, + updated_at: index.updated_at, + }, } } } -impl IndexReader for &V1IndexReader { - type Document = serde_json::Map; - type Settings = settings::Settings; - - fn name(&self) -> &str { - todo!() - } - - fn documents(&self) -> Result>>> { - todo!() - } - - fn settings(&self) -> Result { - todo!() - } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Metadata { + pub indexes: Vec, + pub db_version: String, + pub dump_version: crate::Version, } -impl DumpReader for V1Reader { - type Document = serde_json::Map; - type Settings = settings::Settings; +#[cfg(test)] +pub(crate) mod test { + use std::fs::File; + use std::io::BufReader; - type Task = update::UpdateStatus; - type UpdateFile = Infallible; + use flate2::bufread::GzDecoder; + use meili_snap::insta; + use tempfile::TempDir; - type Key = Infallible; + use super::*; - fn date(&self) -> Option { - None - } + #[test] + fn read_dump_v1() { + let dump = File::open("tests/assets/v1.dump").unwrap(); + let dir = TempDir::new().unwrap(); + let mut dump = BufReader::new(dump); + let gz = GzDecoder::new(&mut dump); + let mut archive = tar::Archive::new(gz); + archive.unpack(dir.path()).unwrap(); - fn version(&self) -> Version { - Version::V1 - } + let dump = V1Reader::open(dir).unwrap(); - fn indexes( - &self, - ) -> Result< - Box< - dyn Iterator< - Item = Result< - Box< - dyn super::IndexReader< - Document = Self::Document, - Settings = Self::Settings, - >, - >, - >, - >, - >, - > { - Ok(Box::new(self.indexes.iter().map(|index| { - let index = Box::new(index) - as Box>; - Ok(index) - }))) - } + // top level infos + assert_eq!(dump.date(), None); - fn tasks(&self) -> Box)>>> { - Box::new(std::iter::from_fn(|| { - self.next_update() - .transpose() - .map(|result| result.map(|task| (task, None))) - })) - } + // indexes + let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); - fn keys(&self) -> Box>> { - Box::new(std::iter::empty()) + let mut products = indexes.pop().unwrap(); + let mut movies = indexes.pop().unwrap(); + let mut dnd_spells = indexes.pop().unwrap(); + + assert!(indexes.is_empty()); + + // products + insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + { + "uid": "products", + "primaryKey": "sku", + "createdAt": "[now]", + "updatedAt": "[now]" + } + "###); + + insta::assert_json_snapshot!(products.settings().unwrap()); + let documents = products.documents().unwrap().collect::>>().unwrap(); + assert_eq!(documents.len(), 10); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); + + // products tasks + let tasks = products.tasks().collect::>>().unwrap(); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"91de507f206ad21964584021932ba7a7"); + + // movies + insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + { + "uid": "movies", + "primaryKey": "id", + "createdAt": "[now]", + "updatedAt": "[now]" + } + "###); + + insta::assert_json_snapshot!(movies.settings().unwrap()); + let documents = movies.documents().unwrap().collect::>>().unwrap(); + assert_eq!(documents.len(), 10); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5"); + + // movies tasks + let tasks = movies.tasks().collect::>>().unwrap(); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"55eef4de2bef7e84c5ce0bee47488f56"); + + // spells + insta::assert_json_snapshot!(dnd_spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + { + "uid": "dnd_spells", + "primaryKey": "index", + "createdAt": "[now]", + "updatedAt": "[now]" + } + "###); + + insta::assert_json_snapshot!(dnd_spells.settings().unwrap()); + let documents = dnd_spells.documents().unwrap().collect::>>().unwrap(); + assert_eq!(documents.len(), 10); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"aa24c0cfc733d66c396237ad44263bed"); + + // spells tasks + let tasks = dnd_spells.tasks().collect::>>().unwrap(); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"836dd7d64d5ad20ad901c44b1b161a4c"); } } diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-10.snap b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-10.snap new file mode 100644 index 000000000..f71df0ae6 --- /dev/null +++ b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-10.snap @@ -0,0 +1,24 @@ +--- +source: dump/src/reader/v1/mod.rs +expression: dnd_spells.settings().unwrap() +--- +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "*" + ], + "displayedAttributes": [ + "*" + ], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [] +} diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-12.snap b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-12.snap new file mode 100644 index 000000000..f71df0ae6 --- /dev/null +++ b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-12.snap @@ -0,0 +1,24 @@ +--- +source: dump/src/reader/v1/mod.rs +expression: dnd_spells.settings().unwrap() +--- +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "*" + ], + "displayedAttributes": [ + "*" + ], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [] +} diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-2.snap b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-2.snap new file mode 100644 index 000000000..b117c5f3d --- /dev/null +++ b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-2.snap @@ -0,0 +1,38 @@ +--- +source: dump/src/reader/v1/mod.rs +expression: products.settings().unwrap() +--- +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "*" + ], + "displayedAttributes": [ + "*" + ], + "stopWords": [], + "synonyms": { + "android": [ + "phone", + "smartphone" + ], + "iphone": [ + "phone", + "smartphone" + ], + "phone": [ + "android", + "iphone", + "smartphone" + ] + }, + "attributesForFaceting": [] +} diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-5.snap b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-5.snap new file mode 100644 index 000000000..aa9ed082a --- /dev/null +++ b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-5.snap @@ -0,0 +1,28 @@ +--- +source: dump/src/reader/v1/mod.rs +expression: movies.settings().unwrap() +--- +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "asc(release_date)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "*" + ], + "displayedAttributes": [ + "*" + ], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [ + "id", + "genres" + ] +} diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-6.snap b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-6.snap new file mode 100644 index 000000000..aa9ed082a --- /dev/null +++ b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-6.snap @@ -0,0 +1,28 @@ +--- +source: dump/src/reader/v1/mod.rs +expression: movies.settings().unwrap() +--- +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "asc(release_date)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "*" + ], + "displayedAttributes": [ + "*" + ], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [ + "id", + "genres" + ] +} diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-7.snap b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-7.snap new file mode 100644 index 000000000..aa9ed082a --- /dev/null +++ b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-7.snap @@ -0,0 +1,28 @@ +--- +source: dump/src/reader/v1/mod.rs +expression: movies.settings().unwrap() +--- +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness", + "asc(release_date)" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "*" + ], + "displayedAttributes": [ + "*" + ], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [ + "id", + "genres" + ] +} diff --git a/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-8.snap b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-8.snap new file mode 100644 index 000000000..f71df0ae6 --- /dev/null +++ b/dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-8.snap @@ -0,0 +1,24 @@ +--- +source: dump/src/reader/v1/mod.rs +expression: dnd_spells.settings().unwrap() +--- +{ + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ], + "distinctAttribute": null, + "searchableAttributes": [ + "*" + ], + "displayedAttributes": [ + "*" + ], + "stopWords": [], + "synonyms": {}, + "attributesForFaceting": [] +} diff --git a/dump/src/reader/v1/update.rs b/dump/src/reader/v1/update.rs index c9ccaf309..b6408f42a 100644 --- a/dump/src/reader/v1/update.rs +++ b/dump/src/reader/v1/update.rs @@ -1,54 +1,8 @@ use serde::{Deserialize, Serialize}; -use serde_json::Value; use time::OffsetDateTime; use super::settings::SettingsUpdate; -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Update { - data: UpdateData, - #[serde(with = "time::serde::rfc3339")] - enqueued_at: OffsetDateTime, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum UpdateData { - ClearAll, - Customs(Vec), - // (primary key, documents) - DocumentsAddition { - primary_key: Option, - documents: Vec>, - }, - DocumentsPartial { - primary_key: Option, - documents: Vec>, - }, - DocumentsDeletion(Vec), - Settings(Box), -} - -impl UpdateData { - pub fn update_type(&self) -> UpdateType { - match self { - UpdateData::ClearAll => UpdateType::ClearAll, - UpdateData::Customs(_) => UpdateType::Customs, - UpdateData::DocumentsAddition { documents, .. } => UpdateType::DocumentsAddition { - number: documents.len(), - }, - UpdateData::DocumentsPartial { documents, .. } => UpdateType::DocumentsPartial { - number: documents.len(), - }, - UpdateData::DocumentsDeletion(deletion) => UpdateType::DocumentsDeletion { - number: deletion.len(), - }, - UpdateData::Settings(update) => UpdateType::Settings { - settings: update.clone(), - }, - } - } -} - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "name")] pub enum UpdateType { diff --git a/dump/src/reader/v1/v1.rs b/dump/src/reader/v1/v1.rs deleted file mode 100644 index 0f4312508..000000000 --- a/dump/src/reader/v1/v1.rs +++ /dev/null @@ -1,22 +0,0 @@ -use serde::Deserialize; -use time::OffsetDateTime; - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Index { - pub name: String, - pub uid: String, - #[serde(with = "time::serde::rfc3339")] - created_at: OffsetDateTime, - #[serde(with = "time::serde::rfc3339")] - updated_at: OffsetDateTime, - pub primary_key: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Metadata { - indexes: Vec, - db_version: String, - dump_version: crate::Version, -} diff --git a/dump/tests/assets/v1.dump b/dump/tests/assets/v1.dump new file mode 100644 index 0000000000000000000000000000000000000000..f1e2959362c60fe8a6fb4ac561fe65037ade8034 GIT binary patch literal 9904 zcmV;hCQsQPiwFP!00000|LtAra@)q1_V0NL45uo~y`%(y;3n6tDO&P2v24eZ?c8Ki zswU7RVSxaH21L`2tL8Q4_2x?R%YjU*XDCCQ{3~_oY|yBE{c-l4~Y25r|Ce zyi-Yh?{+TRvF)zsoxA?Mj<>e{j?;7T_uf~y{^aZbu@T0Bd@ogF@0X8_Sj=Q&uMxz- ziJHqOQjKOK4}KQ;_+7}Vx?g8E?QX%ggqRt=|i@mV1M2N{Jj`a>?Ydsy z-58KM%^RP7Z8n0@8=0ywiE+=~aqS%&C4+o6uWr9_8lSeUf3>dnZtc20{yPr7-QoXN zxT^KvRWi%Mc%p7x6kLY??RIZ{{XM5`-{JpPxIQ*g5uak&U*!=Cfkw8RCxDMdk_L(a zCwU*v!fc7pBFoZnlxMPd{xVTvMgY@n$oIm}VyRSv04Y=;B|jUM753shikb~2Me0w* zD3awP?({FF);tnRg2IhwDof^Xc}RP|(AQ<0#LF4Jef(4#mwJ?>kAyFYj;Y|Mdw18x zSB?LJ#Ls6k&NKi-_ipUEnE#>admH+n=Xt$5{{IS>(f^ zd~%Y7Obia*%Yc0(&mf%$MKlNy`_*0)=!RC(x`sm=} zC-kNKX_7?sAi0?{SN)EsWF}EiEi6GoBP+IBr_EDMrbewA?sgE}BuzkXWPfGFb#N1=OU|d?eBl z%8M~KFTnBvo_&jPXUPl=?r0SVSY)VNvm<8b-_~GMBf}bw9u+qWApJWgXNpc8I|ddH7vku z24QHa-1j9WBq7MdJJD2SOG}9}TDELT$cN4eWHH^{I)>5&9hVD@i7TLYb&0yxqM(hG z$I-#c>UH|g4Z7bqEytyJH#%q~M+;PcS@jbYKh<){uG{7FaFI*U|6~#^v*|)!@hq zzN`=sf2rX~YEX5A=>wT4VzSf{Q4|_zfO3{krevovOJO~e=uDP?lh09rm|V(pFlEPj zNGz)n11t{Z%bF))4D=8%6!RIk>*r|-7OI}A5(EeXItgc)9&$+D+e@;;p|- zuh}GIG1{c#;pi%4T&sBw75foD*01qL@qG&cnzD8Jor_2WVQpM5%Ntu7C`#Wx2{U!x zhF&1H(BCSBY3a?*38)L5>nQ~gh+(`WdnTOI7#8RR{D7LQ15Blew(F)HqnSFkQB0jq z0nD~hOWlsqO53(kO4}Wyl6p2!y3Xp6NM|)-GhgfK1AKP;s`RKdkn(r|aRw2NSxIwp z0GIp?iJl}pfeZAKELe}A1~743R0pgLY?FxQn%I>JTCUw1CzwM&9{~(r$5|LLC+lT{ z<)WIkA%R#kMj@8@m@9N=9npzz2}{P>65}E6STv6ShNRG^Fi_Hqsva(8|C$1VKbCt*aqLBiuT5ORQO;8`V)2*07s~dghME(L=z;$^@xV z1jk%?1mqisy7T(Gp)SqqU;gr!;cq$m_pMm$N`fUdOud*2GCm8_ zBqkhck}oaEGj_{`SaM#|NvbDfAx9(nJCfN#N&`t)Wst7Y7)Wn4BFw~y0Twb{5T)ow zg!a`f;6$TFFdG0Vi7`Yp9&T$PqSNx3d9D2^%aY-I4!^$u>z8i@2#6J}t||B9D3rKQwf~_@4hg9$55Oxs7n3}MpA+4bMi<@o(PBsrf@L*nGctv#b)s_fDohLz53>=b zkd{4WV;jPiy3eMhg|+aN7l3}$s|a2)mtuQ<{C*64{Aw`M)m7x(h&;O zqNZXlsi&op3gE~4MxcC)#zx^>2BMOt=rkt}L#3o)wMP&aU{Wl8kmja5@lnWTG_Bx} zloC)X;QV3^t5}J;(UefKkDi5f`mmU%FrEe6#;oxeT6<3Oft7zN1q75TDUBEax|>)i zA`ZkiiBEuT_&ZHff(H`nwJD5ELkn)GTpu7!Xvi}Ke*jF$K@Am=oXr7LLiTKoK${bp z#2}m|p=s*#Cv>66lp}i5=6sXaaSo0}NT1Xdw!7qd?>s+KKwP<&Ecj7v9Qq**msO4B zmO<@Uk3uNF(XzR%VbWuWaT=1| zKMm(K8PAWbNA*XAsIPp4z+j^>W2xa_^oayVQVV&5{WA%CfRzVJWXcMAY0In?Bpgvv zuDL6o7Jf~sgDq3SP!z|Lv60Sb1(xJfm>h7FOeCWW0!ZtgXg%Vw2qW;;3xe}SsARKh zWQzk8C4ywO;XNHm7AwT9Fji^s1x}W()50-A!=wUuUKba{T9|$y9nVTV2vr-cVuS1^ zohz>QrQQ|i0!JjKt48=wk1ljXQsJvRBC)IxC>d!=wYbj8&GREWwPNS6{%D0~ADqjv zT&~XUU+DXR46b_w56sssm37H_-OsUd){$mATx`LG6}w}Wkg~*vMv_2V+yW{yET~)^ zpTU&F+LHoO9xNFrlvXbac+vU==@z6i>)Mbnfs8`!iJ(p>a8Y5(v9^eao6p0PCb9Tb z>tX7*JcO$=B9{tezqH$-4L2iB{@~YN?L@HAA zbs&yxR8+)p^a_k5rbBMH_lhPswx_TPD$_(;lZ2duLG6*AJ|YzogN17?mD`-ZH|T8o zaslQ3Gl`Es8vk!x)7(V=^$3Ddh^`nvh|kL_*a|Z%p=z&R1CDD%53V5H*b-M8qV1@- zy%N*Yh8iUSDUD4O>tIWq{85$gg5-+x!sD=j3y9qpt8Vb~73Ed8jI5%3x-;!t)1pII z&*%x8wHP+E62tb!?+tQ~$h!h#QDYnGusRLT9AzB5K>ztj!GT2LjbH3SS3or^!EQW2 zfAv~Cx>zoFbV}zS$f}B7WcJ)`ikNa?3(YwpGr9rJ8?HJyxMpAE&YTd9hhH>W+b3H8?2Ts;jt;tLx-;=da z_%nPGp2mrPiUrSONG#=t zNp9x96zwEsDGR$&LeN;zKeUg>vaUF3Ny5aHjpOor&_acylR+) z>!DvW3hTS#)e>+{f|&)nhV!TvTCYY}A40+@M@CxBilk-=HgHS;$OnN^nmly7sEwD> z(-6z=HzPsCn@?A=BFQ-9tZ^-Uw5Z_QGt$t23VQKtxzr3x#mH$evSwGCu~arSh4ONp z(LNU`mMljom2?~}v&a_*2@)h}>8sOC!eMFJGB#5cv?1dykYh5ZXk;8WGT9p-qb#h? zF{w+CEEAZRjhv1mJ4BSW^% zO_;3evviTHtNZJO!1GtPSc@Dq*K1K*BqGBk&<;_ZT9GTcs{NDduZkp&Y8F^C&<>3; z-!FB~DsZBaGFWFeLLZY&Qn;WpZKjYA05EL&;xO;^BGy z-nsIwR`Yh!-~J&3Cad!F3zOiyGV!@)#Bwaq2x#0cXm^!rDLK5B2)8ErF0yRfX<%|a z*BDh9exx9Cj}h7@muD^Y1a7!#vrI^d^$4gLV7bb6w(2d( z=$+Jiuh&$U7v+OOdog$P3xK6bNw7p#hE)i`))Qi}T8Jz#T~ijAX<>V=Js=D)#5os; zFKhjpEDYM+q*WcQYS0Msm~|3bCDUlrnARI*7CBH; zvihjCmBikng3ckSbfOY%x5$coODbSxPiy-qShs>Wb4{)3HkDR!;~GxKzEg4jL@LgI z&}9t^;jCHTIx2N<6#3mcqMVuEttT(>&v2jq(K!!zlBM|fK)$zrr;@E!YwQ2q3I~pU zw_F?lIDP!{*FKGuHz(4zq`$%gW_$*}Q+_hx#O9t z22ju-ciX1AowBYQbWzu->N;G?HI+5c{7%6V^QhLZ8{p1L$Bt);ab*Ue->rUCw_mU8 zuD&l%NSCV)<4RdFP!uw>z?%WH+&9MDaKE|gx8XIx2ds(7+ahi_C^sktSn?FAjCPz6 zK~OFf#s8JHPi;eOV%|b;7_jz)tVR9lrgn-klDC^>vC=(8wRNL92@cZbJ!Qw130dhL$*FBXjJH~#ktxag zGP+aEs*+HGdBZEs-c-Xcl#4ta3twt0*2n=}sUF${_me`ML|IX9AxNPO>+f!Jf(4Co z1aEVoSO6JoJQW~LwLad^6oph-CLA*Y>`}^{&K)a5ZRoow6r}5( zXW#gx(Ja?ZnL-Ia--7ATcAZ~8{oB2}>)fk${)aw?>c&t1xWf5Aj=g#Qk9~Lk&)2v< zHpmg3Q|jD2O?=K3Y7}RZJq>tQbCIN1#3G7wVupqp312vV*VuFHPj&pZcWm41I|D3j zvms-Id-ID4}Nak@%ZOn`V75$w|afX`ERz@bMDT6 z`wEv{|JytJjjqe^-`iaOw%c>>@c%1Zw{ZR&$|?Uln4OeIbwYCF`+uM_;TjxAhq({8 zQ{(P{IK%(9cJ>=x7xO=`-s^1ef7f;H`2QUb2Xd&8hexX8%qA zkkCmYxHZkPx!T*^4f&{_Y!-}ml6110?ap@>aI5dWkCXY+&QICt(&-<*oc`e1{?Eh5 z!xwMlo1J&_37R4o3&?H30iaQ`BwvvB`Le@D@9wB=G2}xWD5Fc|2;RadOgQ@~q*w&3 z##k2d*2>SqnUaJ}5OmW{r|k{~u5Fk6Pp8=xu7Z8^7!b-mq7bTDlI2*;&S21K z*Jo#+cIjd$hax3&_ZpKl%+%!V^dNq6{9CvG%#NO^*Xe$I*gJmrcyp4DIr}e3m&V@> zH>98-#es-9-5g#w#TnN+JX6~QoyVMzF8}X;{ukjo3=aOXg-pkJWYILznJ(mKdkA0u zlzrwRCr^;cEt`cwU{0g5CL$tliccpKl$(dy6pum~^L|#i?~i0z*%=IM_~lloh+VvN zC`8nvxVH$f=sM`H1Bi3@@F?vKuF5=C)P;%Gn?~V4;CV2wXBypyV{69SpkJH!-MZ>d zzfC&E)>R)CBOzZDZK#A>^iimvRs_&--EQBh5x_&`(@xq~Niw_6>i3^1C;Pek+q=gf z#*d%8d+?waPG7|1-q;>(uKrQ@p0;vw-nF8w+MHiMLnP&lvtBJfOo7_kajGhJstn7R zoOXM1xK{7urDuy!`%2*jKEteSn=+tquiagpvg2tQQl>9WS}}W15EV+&3#ZQMc0IRW zo4O~6+n{&2+3*US)OC=GNfPg|Q}PK&533Kc9tGC|pR@Pg-=|~y;LVf%WAE9A=TDF3 zhvP>N4wmi~@KNZ;_YvC#LyUQ=Jb9=Bcw;%enccEx#|vCblGrBTL0UO8YGI*s!Jf0*^-+!rVj-5s=^gM&gIjY- zeialZo9U2*9xU^6iV!cKG;oGtv-^4)$}zDQG#%bN#OLIh(ee>bAw_DL2G}PZ!Ud}o zpA$lXN5xsBL=Ke3qA3LWU~pBk(6Kd54v)kvjFu~0RS|;MvAu4mMhK6|#dw`HR%cHS zj-S6y#Ji(@@a$pIdzh&AXUn0Q{MY6hKji}pq;~C(EyVr-WW1;#RGMN**+m>6A)%wA zp9kVB#J_pWaT!e&EPIO0(^^J_ItU5gg#Y~qAQuQ0VIA^l?x*mCG&vZ{52Ns%m|8E>5Z~oYrW8lg zDUI;{P9>p5MmxW#w}eu5@qQ_12;e0-b%BH{5eSsMtt;H?47&Z*6)yZeBCX3rZ_pXI z_2oQBQXyw1c6~@{<(~CC8HJhnR)O2m4dMM$WsaV_I%kK^UhPLe&VD$0GaQ9^H=XRi z`tbb6cmB~&7fDPCGZkTpSILMHW1b~B?jB`u5yzPM&0!&&4Y9IqUZ6CzJMMXuU(E_B z7=vzoRzMEPbzV1+$1jHd;oJPl@w*qVU;a3sF8r5|9*vKG{IE@nf|Hy`I+AnRVMO~1 z`8X63h6wZo@QkOl2bioYP%|f?kLg5!sf-X!Q+m%o#a$;8y<3E23K#>HlU5Np2G2Jw z7-F$9nQMl>n-;&I-dK&vH9I~_LCvq#~opE7#5Fcj@7WoVuTZNUKaU5~V`y``4D@Jq>pI~WiPT?|1l1tY0SRtmE zJ31iP&?{ra2%(LJh_Wr}Q_b`-Aix$y5?92Dq8%xLRC67Rp&g!Ssr4cLCuT%1q^c)^ z6*)Sa1Ntb9ml78g^ptX5kFtc1)}SO3#F+Iik(@Y%0;)uXtcYsG#ONeUl_t>09c7D9 z>0C;k=V>x+Fc)MX>{t&;)@J2a$8w^1UdIWmvikUCioXZtsfKjsMZ$YwT6#MZr_POy z8cZXS@2O`#06?d#P!sytvJq*evxIY>5NXTI$s-#`;dLF5?YVSY>Q3jYOkStobK5og zeSIx4nZI{~*Uq~i-QoVz!=vek2kqs{LHl(u|KM-Z>XTfJbSRsqdn~QTVx*OhA>^5$ zyrj2bqzKkYc_{RQpxC(+^3E?r-=+zzrhJe1ab`B&zO*Y*rnWY&*X!5U<^k|8k|2)jEYWyjyDy$c2h;TL?~adte$)TyMRz)V z-40ZHbBSJP5Ki;Bu!fejC&wA|drhnCo(%-j563y5xC=CRm&d0|Qfu_#O`K)44;BJz z#G#1I!qFD+9nNn7HZ{w%>RfSpCe1$0tJBvomM+cWvOPj6ExcK+7#U+%vxI+Djju`j z;W1?hpJQpHrnpg>oI(+?v;u6#K=~ZBwVLErlMUS8^3XqpX^7QhS|CQ&M-#%(2dqHc zFC{00y?G1CHQB1!c>T6(f1VY1?_c*a{=Zv`{nB-b|Gx+0dW--6F8=#9uG^3Qw(XwN z^9C2jf6dQy4eVa8+vyB0i~s8408+1qD&3zsBfNYMI-qCmC7g|yI5GjR2E`xD$GtKb za+2z3NaK0JYDS|9Uz}7ykosfuyId!y{KH(;{VKRoRorM;M>x(aKw*v(9gbPKU4}bq zH>)O2fbu8fGVbxQ@p`VI@RtGrn>rM-0+hdgqI1F}dv*5oU(2in;Xbzq=%>@5_JH|k zkS2ju1P`pTcdO=WwS@1>Ip+}4spfl>z^V5O1=cV6^RErl=dnId(lmUM(gtyrok-(l z;=lVzP%pznDxN(H#lY0gOu}T>%di7K=`I&k|FK?wm1u1WKn}VW6K!wL?(cMa zeP|JV_tK;VQ+&|&1}<8^oM_$8B-;8?>+smGpYG`QPe#9Prvrc6W%TRWJFe%o2YuTa zT!DV6INI%ZdxQSv^xOVi(A_6J?+$YRbR>&7u-(3lWIKJz6LVqwxb_uDmWub=-Ja9+ zoXbhpy8*~O*7*}}P5#8oTK>eamOt@!Eq~%+l|S(j9wZ-quaL>7I}ra9a>T%PE&=hu zp5yFvzz=qBK&X5PN9f|7twYzuzzprrp^Z)*H13FxqP;lm##(YZ0KjSeZ zex|>maHv0>`@DQyv{Y7EE%~qbOVcI)-(m*vCHcQy_!Ktxe|7JE|M_cN(DhDp7{2WR zg~keXE~m6kL`9+lZOMzto_8x|y6L2(0oBSqi7l{TXgz-PoRQ*U~tk3#mFyLZY zuR-uM4f*)VVm4}rG4qSf=1}S>8G-}iN^&j*|C=Up=pz_@hPhPtdHN}l=e~9D`ql8+ zi~S$0my|zYy`;3Tlrr|DKaGziBMzD8B`I0SFI+A=hFb%YX5Uo-1sx95$E)&nr{6E9sZ!k`r zb|2~X~Ag!mt-}v!2{%roYSBF3C?-RP}Ur8;x3}=&_(P+sh;`!X^0}S9nzC3w- zq(kUxx5O!GP4{@B1P9C6U|}NeciQI=u?3^|jkTZ5=Q2G`=3h?IP6rrvN74mJKTG0C zE7IHWEjna|$nh#9%O~-Z4+d>=rt}B>enIi{_~s}ct?3>&DBf{<-G1*JlCRNt z|8g4dZP0k1cH???H2ztzeGSy^UkA0BqWjmU=!%4Oce|Q&R#Dr7)2M4`SUJk1_&fxt`L91Ai zs}r_+EyC{F-UIiVggrQ`QMT@GSCi8!Wp@yJDkxh&zcI?D=1iZPlnvVGdlhAesivui z_OYLdqh~>i|8F{v$KWXMLG^G zI&7SF!@P*2xA@(HXzZSNBMpZ6P|bFF}2_Yl9YcK>%RU)%Nn(9odNyR_6qYY zXCOQJrH6#w^V&7QUv;#bexFD3rYgO^MMTRFv`-;RbKdiM-#k{dy+CHxglCg}0Q*#J zA;T?$sgWe+@%U=gXxeF2Or%<8w5TmXi`wF0w0IaT#f!SDDZ>z{GLG@*NC*;c6x)YrrL9BpMdz`z`iC9cea>ov{mC@2Zy^=l_NS|Q$*SFT0lJ9 zVx^Hs^QtD94U^2hrb!mJ7&Ya<-U6FU1_ed3_p1XxSVfWi{N^YU&6z$oVH1i-uW}w7 z$}AP2##iJ;WQ-C~q<`z{K&YwpL#Vd~9@3c@&Y(YN(WW2^(Y&B(J%g#6L@_AmP^La@ z&wnGQp-(>x*|Hu)T0Wgi$4<9S$o;Mdl(*epXSxwoyEW6V+$vtVHHw(!na>Q%yG|c$--E*t%8_AtetvT-kLFCDl@Pjj zcir?=PWUbE0K6ps2bPz6cmCHObbXEef4~{Le{s%gTbn0#r)wkZbZO42DGs9Qcy{+n zS2A-`svdJY0c#@H?x@Sp4aVp?!^?*+v|8cZMW}rVU}OI iIms0F+5>2!ox2?3yX)?{yY8+_um1;EI63+NumAwwt69AO literal 0 HcmV?d00001 From c8841344e29435d9d419321baaa723be2aee3027 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 30 Nov 2022 18:17:54 +0100 Subject: [PATCH 09/17] v1: RankingRule::from_str --- dump/src/reader/v1/settings.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/dump/src/reader/v1/settings.rs b/dump/src/reader/v1/settings.rs index 0065d3f97..2f7976534 100644 --- a/dump/src/reader/v1/settings.rs +++ b/dump/src/reader/v1/settings.rs @@ -1,6 +1,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::result::Result as StdResult; +use std::str::FromStr; +use once_cell::sync::Lazy; +use regex::Regex; use serde::{Deserialize, Deserializer, Serialize}; #[derive(Default, Clone, Serialize, Deserialize, Debug)] @@ -53,6 +56,34 @@ pub enum RankingRule { Desc(String), } +static ASC_DESC_REGEX: Lazy = + Lazy::new(|| Regex::new(r#"(asc|desc)\(([\w_-]+)\)"#).unwrap()); + +impl FromStr for RankingRule { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(match s { + "typo" => Self::Typo, + "words" => Self::Words, + "proximity" => Self::Proximity, + "attribute" => Self::Attribute, + "wordsPosition" => Self::WordsPosition, + "exactness" => Self::Exactness, + text => { + let caps = ASC_DESC_REGEX.captures(text).ok_or(())?; + let order = caps.get(1).unwrap().as_str(); + let field_name = caps.get(2).unwrap().as_str(); + match order { + "asc" => Self::Asc(field_name.to_string()), + "desc" => Self::Desc(field_name.to_string()), + _ => return Err(()), + } + } + }) + } +} + // Any value that is present is considered Some value, including null. fn deserialize_some<'de, T, D>(deserializer: D) -> StdResult, D::Error> where From c7749127fa332b949a1b0df16a4ff3f8a3ed4627 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 30 Nov 2022 14:55:45 +0100 Subject: [PATCH 10/17] Use reader v1 and compat to v2 --- dump/src/error.rs | 3 - dump/src/reader/compat/mod.rs | 1 + dump/src/reader/compat/v1_to_v2.rs | 325 +++++++++++++++++++++++++++++ dump/src/reader/compat/v2_to_v3.rs | 77 +++++-- dump/src/reader/mod.rs | 9 +- 5 files changed, 386 insertions(+), 29 deletions(-) create mode 100644 dump/src/reader/compat/v1_to_v2.rs diff --git a/dump/src/error.rs b/dump/src/error.rs index a11aae9cf..0d57729ae 100644 --- a/dump/src/error.rs +++ b/dump/src/error.rs @@ -3,8 +3,6 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum Error { - #[error("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.")] - DumpV1Unsupported, #[error("Bad index name.")] BadIndexName, #[error("Malformed task.")] @@ -28,7 +26,6 @@ impl ErrorCode for Error { Error::Uuid(_) => Code::Internal, // all these errors should never be raised when creating a dump, thus no error code should be associated. - Error::DumpV1Unsupported => Code::Internal, Error::BadIndexName => Code::Internal, Error::MalformedTask => Code::Internal, } diff --git a/dump/src/reader/compat/mod.rs b/dump/src/reader/compat/mod.rs index 29836aa61..aabf400d9 100644 --- a/dump/src/reader/compat/mod.rs +++ b/dump/src/reader/compat/mod.rs @@ -1,3 +1,4 @@ +pub mod v1_to_v2; pub mod v2_to_v3; pub mod v3_to_v4; pub mod v4_to_v5; diff --git a/dump/src/reader/compat/v1_to_v2.rs b/dump/src/reader/compat/v1_to_v2.rs new file mode 100644 index 000000000..d8713a344 --- /dev/null +++ b/dump/src/reader/compat/v1_to_v2.rs @@ -0,0 +1,325 @@ +use std::{collections::BTreeSet, str::FromStr}; + +use crate::reader::{v1, v2, Document}; + +use super::v2_to_v3::CompatV2ToV3; +use crate::Result; + +pub struct CompatV1ToV2 { + pub from: v1::V1Reader, +} + +impl CompatV1ToV2 { + pub fn new(v1: v1::V1Reader) -> Self { + Self { from: v1 } + } + + pub fn to_v3(self) -> CompatV2ToV3 { + CompatV2ToV3::Compat(self) + } + + pub fn version(&self) -> crate::Version { + self.from.version() + } + + pub fn date(&self) -> Option { + self.from.date() + } + + pub fn index_uuid(&self) -> Vec { + self.from + .index_uuid() + .into_iter() + .enumerate() + // we use the index of the index 😬 as UUID for the index, so that we can link the v2::Task to their index + .map(|(index, index_uuid)| v2::meta::IndexUuid { + uid: index_uuid.uid, + uuid: uuid::Uuid::from_u128(index as u128), + }) + .collect() + } + + pub fn indexes(&self) -> Result> + '_> { + Ok(self.from.indexes()?.map(|index_reader| Ok(CompatIndexV1ToV2 { from: index_reader? }))) + } + + pub fn tasks( + &mut self, + ) -> Box)>> + '_> { + // Convert an error here to an iterator yielding the error + let indexes = match self.from.indexes() { + Ok(indexes) => indexes, + Err(err) => return Box::new(std::iter::once(Err(err))), + }; + let it = indexes.enumerate().flat_map( + move |(index, index_reader)| -> Box> { + let index_reader = match index_reader { + Ok(index_reader) => index_reader, + Err(err) => return Box::new(std::iter::once(Err(err))), + }; + Box::new( + index_reader + .tasks() + // Filter out the UpdateStatus::Customs variant that is not supported in v2 + // and enqueued tasks, that don't contain the necessary update file in v1 + .filter_map(move |task| -> Option<_> { + let task = match task { + Ok(task) => task, + Err(err) => return Some(Err(err)), + }; + Some(Ok(( + v2::Task { + uuid: uuid::Uuid::from_u128(index as u128), + update: Option::from(task)?, + }, + None, + ))) + }), + ) + }, + ); + Box::new(it) + } +} + +pub struct CompatIndexV1ToV2 { + pub from: v1::V1IndexReader, +} + +impl CompatIndexV1ToV2 { + pub fn metadata(&self) -> &crate::IndexMetadata { + self.from.metadata() + } + + pub fn documents(&mut self) -> Result> + '_>> { + self.from.documents().map(|it| Box::new(it) as Box>) + } + + pub fn settings(&mut self) -> Result> { + Ok(v2::settings::Settings::::from(self.from.settings()?).check()) + } +} + +impl From for v2::Settings { + fn from(source: v1::settings::Settings) -> Self { + let displayed_attributes = source + .displayed_attributes + .map(|opt| opt.map(|displayed_attributes| displayed_attributes.into_iter().collect())); + let attributes_for_faceting = source.attributes_for_faceting.map(|opt| { + opt.map(|attributes_for_faceting| attributes_for_faceting.into_iter().collect()) + }); + let ranking_rules = source.ranking_rules.map(|opt| { + opt.map(|ranking_rules| { + ranking_rules + .into_iter() + .filter_map(|ranking_rule| { + match v1::settings::RankingRule::from_str(&ranking_rule) { + Ok(ranking_rule) => { + let criterion: Option = + ranking_rule.into(); + criterion.as_ref().map(ToString::to_string) + } + Err(()) => Some(ranking_rule), + } + }) + .collect() + }) + }); + + Self { + displayed_attributes, + searchable_attributes: source.searchable_attributes, + filterable_attributes: attributes_for_faceting, + ranking_rules, + stop_words: source.stop_words, + synonyms: source.synonyms, + distinct_attribute: source.distinct_attribute, + _kind: std::marker::PhantomData, + } + } +} + +impl From for Option { + fn from(source: v1::update::UpdateStatus) -> Self { + use v1::update::UpdateStatus as UpdateStatusV1; + use v2::updates::UpdateStatus as UpdateStatusV2; + Some(match source { + UpdateStatusV1::Enqueued { content } => { + log::warn!( + "Cannot import task {} (importing enqueued tasks from v1 dumps is unsupported)", + content.update_id + ); + log::warn!("Task will be skipped in the queue of imported tasks."); + + return None; + } + UpdateStatusV1::Failed { content } => UpdateStatusV2::Failed(v2::updates::Failed { + from: v2::updates::Processing { + from: v2::updates::Enqueued { + update_id: content.update_id, + meta: Option::from(content.update_type)?, + enqueued_at: content.enqueued_at, + content: None, + }, + started_processing_at: content.processed_at + - std::time::Duration::from_secs_f64(content.duration), + }, + error: v2::ResponseError { + // error code is ignored by serialization, and so always default in deserialized v2 dumps + // that's a good thing, because we don't have them in v1 dump 😅 + code: http::StatusCode::default(), + message: content.error.unwrap_or_default(), + // error codes are unchanged between v1 and v2 + error_code: content.error_code.unwrap_or_default(), + // error types are unchanged between v1 and v2 + error_type: content.error_type.unwrap_or_default(), + // error links are unchanged between v1 and v2 + error_link: content.error_link.unwrap_or_default(), + }, + failed_at: content.processed_at, + }), + UpdateStatusV1::Processed { content } => { + UpdateStatusV2::Processed(v2::updates::Processed { + success: match &content.update_type { + v1::update::UpdateType::ClearAll => { + v2::updates::UpdateResult::DocumentDeletion { deleted: u64::MAX } + } + v1::update::UpdateType::Customs => v2::updates::UpdateResult::Other, + v1::update::UpdateType::DocumentsAddition { number } => { + v2::updates::UpdateResult::DocumentsAddition( + v2::updates::DocumentAdditionResult { nb_documents: *number }, + ) + } + v1::update::UpdateType::DocumentsPartial { number } => { + v2::updates::UpdateResult::DocumentsAddition( + v2::updates::DocumentAdditionResult { nb_documents: *number }, + ) + } + v1::update::UpdateType::DocumentsDeletion { number } => { + v2::updates::UpdateResult::DocumentDeletion { deleted: *number as u64 } + } + v1::update::UpdateType::Settings { .. } => v2::updates::UpdateResult::Other, + }, + processed_at: content.processed_at, + from: v2::updates::Processing { + from: v2::updates::Enqueued { + update_id: content.update_id, + meta: Option::from(content.update_type)?, + enqueued_at: content.enqueued_at, + content: None, + }, + started_processing_at: content.processed_at + - std::time::Duration::from_secs_f64(content.duration), + }, + }) + } + }) + } +} + +impl From for Option { + fn from(source: v1::update::UpdateType) -> Self { + Some(match source { + v1::update::UpdateType::ClearAll => v2::updates::UpdateMeta::ClearDocuments, + v1::update::UpdateType::Customs => { + log::warn!("Ignoring task with type 'Customs' that is no longer supported"); + return None; + } + v1::update::UpdateType::DocumentsAddition { .. } => { + v2::updates::UpdateMeta::DocumentsAddition { + method: v2::updates::IndexDocumentsMethod::ReplaceDocuments, + format: v2::updates::UpdateFormat::Json, + primary_key: None, + } + } + v1::update::UpdateType::DocumentsPartial { .. } => { + v2::updates::UpdateMeta::DocumentsAddition { + method: v2::updates::IndexDocumentsMethod::UpdateDocuments, + format: v2::updates::UpdateFormat::Json, + primary_key: None, + } + } + v1::update::UpdateType::DocumentsDeletion { .. } => { + v2::updates::UpdateMeta::DeleteDocuments { ids: vec![] } + } + v1::update::UpdateType::Settings { settings } => { + v2::updates::UpdateMeta::Settings((*settings).into()) + } + }) + } +} + +impl From for v2::Settings { + fn from(source: v1::settings::SettingsUpdate) -> Self { + let displayed_attributes: Option>> = + source.displayed_attributes.into(); + + let attributes_for_faceting: Option>> = + source.attributes_for_faceting.into(); + + let ranking_rules: Option>> = + source.ranking_rules.into(); + + // go from the concrete types of v1 (RankingRule) to the concrete type of v2 (Criterion), + // and then back to string as this is what the settings manipulate + let ranking_rules = ranking_rules.map(|opt| { + opt.map(|ranking_rules| { + ranking_rules + .into_iter() + // filter out the WordsPosition ranking rule that exists in v1 but not v2 + .filter_map(|ranking_rule| { + Option::::from(ranking_rule) + }) + .map(|criterion| criterion.to_string()) + .collect() + }) + }); + + Self { + displayed_attributes: displayed_attributes.map(|opt| { + opt.map(|displayed_attributes| displayed_attributes.into_iter().collect()) + }), + searchable_attributes: source.searchable_attributes.into(), + filterable_attributes: attributes_for_faceting.map(|opt| { + opt.map(|attributes_for_faceting| attributes_for_faceting.into_iter().collect()) + }), + ranking_rules, + stop_words: source.stop_words.into(), + synonyms: source.synonyms.into(), + distinct_attribute: source.distinct_attribute.into(), + _kind: std::marker::PhantomData, + } + } +} + +impl From for Option { + fn from(source: v1::settings::RankingRule) -> Self { + match source { + v1::settings::RankingRule::Typo => Some(v2::settings::Criterion::Typo), + v1::settings::RankingRule::Words => Some(v2::settings::Criterion::Words), + v1::settings::RankingRule::Proximity => Some(v2::settings::Criterion::Proximity), + v1::settings::RankingRule::Attribute => Some(v2::settings::Criterion::Attribute), + v1::settings::RankingRule::WordsPosition => { + log::warn!("Removing the 'WordsPosition' ranking rule that is no longer supported, please check the resulting ranking rules of your indexes"); + None + } + v1::settings::RankingRule::Exactness => Some(v2::settings::Criterion::Exactness), + v1::settings::RankingRule::Asc(field_name) => { + Some(v2::settings::Criterion::Asc(field_name)) + } + v1::settings::RankingRule::Desc(field_name) => { + Some(v2::settings::Criterion::Desc(field_name)) + } + } + } +} + +impl From> for Option> { + fn from(source: v1::settings::UpdateState) -> Self { + match source { + v1::settings::UpdateState::Update(new_value) => Some(Some(new_value)), + v1::settings::UpdateState::Clear => Some(None), + v1::settings::UpdateState::Nothing => None, + } + } +} diff --git a/dump/src/reader/compat/v2_to_v3.rs b/dump/src/reader/compat/v2_to_v3.rs index 70bc5b867..8574e04b4 100644 --- a/dump/src/reader/compat/v2_to_v3.rs +++ b/dump/src/reader/compat/v2_to_v3.rs @@ -4,22 +4,28 @@ use std::str::FromStr; use time::OffsetDateTime; use uuid::Uuid; +use super::v1_to_v2::{CompatIndexV1ToV2, CompatV1ToV2}; use super::v3_to_v4::CompatV3ToV4; use crate::reader::{v2, v3, Document}; use crate::Result; -pub struct CompatV2ToV3 { - pub from: v2::V2Reader, +pub enum CompatV2ToV3 { + V2(v2::V2Reader), + Compat(CompatV1ToV2), } impl CompatV2ToV3 { pub fn new(v2: v2::V2Reader) -> CompatV2ToV3 { - CompatV2ToV3 { from: v2 } + CompatV2ToV3::V2(v2) } pub fn index_uuid(&self) -> Vec { - self.from - .index_uuid() + let v2_uuids = match self { + CompatV2ToV3::V2(from) => from.index_uuid(), + CompatV2ToV3::Compat(compat) => compat.index_uuid(), + }; + v2_uuids + .into_iter() .into_iter() .map(|index| v3::meta::IndexUuid { uid: index.uid, uuid: index.uuid }) .collect() @@ -30,11 +36,17 @@ impl CompatV2ToV3 { } pub fn version(&self) -> crate::Version { - self.from.version() + match self { + CompatV2ToV3::V2(from) => from.version(), + CompatV2ToV3::Compat(compat) => compat.version(), + } } pub fn date(&self) -> Option { - self.from.date() + match self { + CompatV2ToV3::V2(from) => from.date(), + CompatV2ToV3::Compat(compat) => compat.date(), + } } pub fn instance_uid(&self) -> Result> { @@ -42,10 +54,18 @@ impl CompatV2ToV3 { } pub fn indexes(&self) -> Result> + '_> { - Ok(self.from.indexes()?.map(|index_reader| -> Result<_> { - let compat = CompatIndexV2ToV3::new(index_reader?); - Ok(compat) - })) + Ok(match self { + CompatV2ToV3::V2(from) => Box::new(from.indexes()?.map(|index_reader| -> Result<_> { + let compat = CompatIndexV2ToV3::new(index_reader?); + Ok(compat) + })) + as Box> + '_>, + CompatV2ToV3::Compat(compat) => Box::new(compat.indexes()?.map(|index_reader| { + let compat = CompatIndexV2ToV3::Compat(Box::new(index_reader?)); + Ok(compat) + })) + as Box> + '_>, + }) } pub fn tasks( @@ -54,11 +74,13 @@ impl CompatV2ToV3 { dyn Iterator>>>)>> + '_, > { - let _indexes = self.from.index_uuid.clone(); + let tasks = match self { + CompatV2ToV3::V2(from) => from.tasks(), + CompatV2ToV3::Compat(compat) => compat.tasks(), + }; Box::new( - self.from - .tasks() + tasks .map(move |task| { task.map(|(task, content_file)| { let task = v3::Task { uuid: task.uuid, update: task.update.into() }; @@ -76,27 +98,38 @@ impl CompatV2ToV3 { } } -pub struct CompatIndexV2ToV3 { - from: v2::V2IndexReader, +pub enum CompatIndexV2ToV3 { + V2(v2::V2IndexReader), + Compat(Box), } impl CompatIndexV2ToV3 { pub fn new(v2: v2::V2IndexReader) -> CompatIndexV2ToV3 { - CompatIndexV2ToV3 { from: v2 } + CompatIndexV2ToV3::V2(v2) } pub fn metadata(&self) -> &crate::IndexMetadata { - self.from.metadata() + match self { + CompatIndexV2ToV3::V2(from) => from.metadata(), + CompatIndexV2ToV3::Compat(compat) => compat.metadata(), + } } pub fn documents(&mut self) -> Result> + '_>> { - self.from - .documents() - .map(|iter| Box::new(iter) as Box> + '_>) + match self { + CompatIndexV2ToV3::V2(from) => from + .documents() + .map(|iter| Box::new(iter) as Box> + '_>), + CompatIndexV2ToV3::Compat(compat) => compat.documents(), + } } pub fn settings(&mut self) -> Result> { - Ok(v3::Settings::::from(self.from.settings()?).check()) + let settings = match self { + CompatIndexV2ToV3::V2(from) => from.settings()?, + CompatIndexV2ToV3::Compat(compat) => compat.settings()?, + }; + Ok(v3::Settings::::from(settings).check()) } } diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index d1ca9ec42..1368596d0 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -9,11 +9,11 @@ use self::compat::v4_to_v5::CompatV4ToV5; use self::compat::v5_to_v6::{CompatIndexV5ToV6, CompatV5ToV6}; use self::v5::V5Reader; use self::v6::{V6IndexReader, V6Reader}; -use crate::{Error, Result, Version}; +use crate::{Result, Version}; mod compat; -// pub(self) mod v1; +pub(self) mod v1; pub(self) mod v2; pub(self) mod v3; pub(self) mod v4; @@ -45,8 +45,9 @@ impl DumpReader { let MetadataVersion { dump_version } = serde_json::from_reader(&mut meta_file)?; match dump_version { - // Version::V1 => Ok(Box::new(v1::Reader::open(path)?)), - Version::V1 => Err(Error::DumpV1Unsupported), + Version::V1 => { + Ok(v1::V1Reader::open(path)?.to_v2().to_v3().to_v4().to_v5().to_v6().into()) + } Version::V2 => Ok(v2::V2Reader::open(path)?.to_v3().to_v4().to_v5().to_v6().into()), Version::V3 => Ok(v3::V3Reader::open(path)?.to_v4().to_v5().to_v6().into()), Version::V4 => Ok(v4::V4Reader::open(path)?.to_v5().to_v6().into()), From e03c2169522e44db85e1fcbc8e811d7310b35414 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 1 Dec 2022 11:46:16 +0100 Subject: [PATCH 11/17] Add compat_v1_to_v2 test --- ...mpat__v1_to_v2__test__compat_v1_v2-11.snap | 23 +++++ ...ompat__v1_to_v2__test__compat_v1_v2-3.snap | 37 ++++++++ ...ompat__v1_to_v2__test__compat_v1_v2-6.snap | 27 ++++++ ...ompat__v1_to_v2__test__compat_v1_v2-7.snap | 27 ++++++ ...ompat__v1_to_v2__test__compat_v1_v2-9.snap | 23 +++++ dump/src/reader/compat/v1_to_v2.rs | 89 +++++++++++++++++++ 6 files changed, 226 insertions(+) create mode 100644 dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-11.snap create mode 100644 dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap create mode 100644 dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap create mode 100644 dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-7.snap create mode 100644 dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-11.snap b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-11.snap new file mode 100644 index 000000000..1adf85e6a --- /dev/null +++ b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-11.snap @@ -0,0 +1,23 @@ +--- +source: dump/src/reader/compat/v1_to_v2.rs +expression: spells.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness" + ], + "stopWords": [], + "synonyms": {}, + "distinctAttribute": null +} diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap new file mode 100644 index 000000000..f7e1736b1 --- /dev/null +++ b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap @@ -0,0 +1,37 @@ +--- +source: dump/src/reader/compat/v1_to_v2.rs +expression: products.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness" + ], + "stopWords": [], + "synonyms": { + "android": [ + "phone", + "smartphone" + ], + "iphone": [ + "phone", + "smartphone" + ], + "phone": [ + "android", + "iphone", + "smartphone" + ] + }, + "distinctAttribute": null +} diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap new file mode 100644 index 000000000..8c36fe96c --- /dev/null +++ b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap @@ -0,0 +1,27 @@ +--- +source: dump/src/reader/compat/v1_to_v2.rs +expression: movies.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [ + "genres", + "id" + ], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness", + "asc(release_date)" + ], + "stopWords": [], + "synonyms": {}, + "distinctAttribute": null +} diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-7.snap b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-7.snap new file mode 100644 index 000000000..8c36fe96c --- /dev/null +++ b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-7.snap @@ -0,0 +1,27 @@ +--- +source: dump/src/reader/compat/v1_to_v2.rs +expression: movies.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [ + "genres", + "id" + ], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness", + "asc(release_date)" + ], + "stopWords": [], + "synonyms": {}, + "distinctAttribute": null +} diff --git a/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap new file mode 100644 index 000000000..1adf85e6a --- /dev/null +++ b/dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap @@ -0,0 +1,23 @@ +--- +source: dump/src/reader/compat/v1_to_v2.rs +expression: spells.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness" + ], + "stopWords": [], + "synonyms": {}, + "distinctAttribute": null +} diff --git a/dump/src/reader/compat/v1_to_v2.rs b/dump/src/reader/compat/v1_to_v2.rs index d8713a344..23e4529dc 100644 --- a/dump/src/reader/compat/v1_to_v2.rs +++ b/dump/src/reader/compat/v1_to_v2.rs @@ -323,3 +323,92 @@ impl From> for Option> { } } } + +#[cfg(test)] +pub(crate) mod test { + use std::fs::File; + use std::io::BufReader; + + use flate2::bufread::GzDecoder; + use meili_snap::insta; + use tempfile::TempDir; + + use super::*; + + #[test] + fn compat_v1_v2() { + let dump = File::open("tests/assets/v1.dump").unwrap(); + let dir = TempDir::new().unwrap(); + let mut dump = BufReader::new(dump); + let gz = GzDecoder::new(&mut dump); + let mut archive = tar::Archive::new(gz); + archive.unpack(dir.path()).unwrap(); + + let mut dump = v1::V1Reader::open(dir).unwrap().to_v2(); + + // top level infos + assert_eq!(dump.date(), None); + + // tasks + let tasks = dump.tasks().collect::>>().unwrap(); + let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"ad6245d98d1a8e30535f3339a9a8d223"); + assert_eq!(update_files.len(), 9); + assert!(update_files[..].iter().all(|u| u.is_none())); // no update file in dumps v1 + + // indexes + let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); + // the index are not ordered in any way by default + indexes.sort_by_key(|index| index.metadata().uid.to_string()); + + let mut products = indexes.pop().unwrap(); + let mut movies = indexes.pop().unwrap(); + let mut spells = indexes.pop().unwrap(); + assert!(indexes.is_empty()); + + // products + insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + { + "uid": "products", + "primaryKey": "sku", + "createdAt": "[now]", + "updatedAt": "[now]" + } + "###); + + insta::assert_json_snapshot!(products.settings().unwrap()); + let documents = products.documents().unwrap().collect::>>().unwrap(); + assert_eq!(documents.len(), 10); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); + + // movies + insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + { + "uid": "movies", + "primaryKey": "id", + "createdAt": "[now]", + "updatedAt": "[now]" + } + "###); + + insta::assert_json_snapshot!(movies.settings().unwrap()); + let documents = movies.documents().unwrap().collect::>>().unwrap(); + assert_eq!(documents.len(), 10); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5"); + + // spells + insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + { + "uid": "dnd_spells", + "primaryKey": "index", + "createdAt": "[now]", + "updatedAt": "[now]" + } + "###); + + insta::assert_json_snapshot!(spells.settings().unwrap()); + let documents = spells.documents().unwrap().collect::>>().unwrap(); + assert_eq!(documents.len(), 10); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"aa24c0cfc733d66c396237ad44263bed"); + } +} From 1260c32d1879ccd74143ab21ad35a5eda49067ed Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 1 Dec 2022 14:01:52 +0100 Subject: [PATCH 12/17] Add reader mod test --- dump/src/reader/mod.rs | 77 +++++++++++++++++++ ...dump__reader__test__import_dump_v1-10.snap | 27 +++++++ ...dump__reader__test__import_dump_v1-11.snap | 23 ++++++ ...dump__reader__test__import_dump_v1-14.snap | 23 ++++++ .../dump__reader__test__import_dump_v1-5.snap | 37 +++++++++ .../dump__reader__test__import_dump_v1-6.snap | 37 +++++++++ .../dump__reader__test__import_dump_v1-8.snap | 27 +++++++ 7 files changed, 251 insertions(+) create mode 100644 dump/src/reader/snapshots/dump__reader__test__import_dump_v1-10.snap create mode 100644 dump/src/reader/snapshots/dump__reader__test__import_dump_v1-11.snap create mode 100644 dump/src/reader/snapshots/dump__reader__test__import_dump_v1-14.snap create mode 100644 dump/src/reader/snapshots/dump__reader__test__import_dump_v1-5.snap create mode 100644 dump/src/reader/snapshots/dump__reader__test__import_dump_v1-6.snap create mode 100644 dump/src/reader/snapshots/dump__reader__test__import_dump_v1-8.snap diff --git a/dump/src/reader/mod.rs b/dump/src/reader/mod.rs index 1368596d0..efbca06d0 100644 --- a/dump/src/reader/mod.rs +++ b/dump/src/reader/mod.rs @@ -529,4 +529,81 @@ pub(crate) mod test { assert_eq!(documents.len(), 10); meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce"); } + + #[test] + fn import_dump_v1() { + let dump = File::open("tests/assets/v1.dump").unwrap(); + let mut dump = DumpReader::open(dump).unwrap(); + + // top level infos + assert_eq!(dump.date(), None); + assert_eq!(dump.instance_uid().unwrap(), None); + + // tasks + let tasks = dump.tasks().unwrap().collect::>>().unwrap(); + let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip(); + meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"b3e3652bfc10a76670be157d2507d761"); + assert_eq!(update_files.len(), 9); + assert!(update_files[..].iter().all(|u| u.is_none())); // no update file in dump v1 + + // keys + let keys = dump.keys().unwrap().collect::>>().unwrap(); + meili_snap::snapshot!(meili_snap::json_string!(keys), @"[]"); + meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"d751713988987e9331980363e24189ce"); + + // indexes + let mut indexes = dump.indexes().unwrap().collect::>>().unwrap(); + // the index are not ordered in any way by default + indexes.sort_by_key(|index| index.metadata().uid.to_string()); + + let mut products = indexes.pop().unwrap(); + let mut movies = indexes.pop().unwrap(); + let mut spells = indexes.pop().unwrap(); + assert!(indexes.is_empty()); + + // products + insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + { + "uid": "products", + "primaryKey": "sku", + "createdAt": "[now]", + "updatedAt": "[now]" + } + "###); + + insta::assert_json_snapshot!(products.settings().unwrap()); + let documents = products.documents().unwrap().collect::>>().unwrap(); + assert_eq!(documents.len(), 10); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca"); + + // movies + insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + { + "uid": "movies", + "primaryKey": "id", + "createdAt": "[now]", + "updatedAt": "[now]" + } + "###); + + insta::assert_json_snapshot!(movies.settings().unwrap()); + let documents = movies.documents().unwrap().collect::>>().unwrap(); + assert_eq!(documents.len(), 10); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5"); + + // spells + insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###" + { + "uid": "dnd_spells", + "primaryKey": "index", + "createdAt": "[now]", + "updatedAt": "[now]" + } + "###); + + insta::assert_json_snapshot!(spells.settings().unwrap()); + let documents = spells.documents().unwrap().collect::>>().unwrap(); + assert_eq!(documents.len(), 10); + meili_snap::snapshot_hash!(format!("{:#?}", documents), @"aa24c0cfc733d66c396237ad44263bed"); + } } diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-10.snap b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-10.snap new file mode 100644 index 000000000..d20fdc77e --- /dev/null +++ b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-10.snap @@ -0,0 +1,27 @@ +--- +source: dump/src/reader/mod.rs +expression: movies.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [ + "genres", + "id" + ], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness", + "release_date:asc" + ], + "stopWords": [], + "synonyms": {}, + "distinctAttribute": null +} diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-11.snap b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-11.snap new file mode 100644 index 000000000..997d303e7 --- /dev/null +++ b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-11.snap @@ -0,0 +1,23 @@ +--- +source: dump/src/reader/mod.rs +expression: spells.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness" + ], + "stopWords": [], + "synonyms": {}, + "distinctAttribute": null +} diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-14.snap b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-14.snap new file mode 100644 index 000000000..997d303e7 --- /dev/null +++ b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-14.snap @@ -0,0 +1,23 @@ +--- +source: dump/src/reader/mod.rs +expression: spells.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness" + ], + "stopWords": [], + "synonyms": {}, + "distinctAttribute": null +} diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-5.snap b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-5.snap new file mode 100644 index 000000000..282cd6ba7 --- /dev/null +++ b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-5.snap @@ -0,0 +1,37 @@ +--- +source: dump/src/reader/mod.rs +expression: products.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness" + ], + "stopWords": [], + "synonyms": { + "android": [ + "phone", + "smartphone" + ], + "iphone": [ + "phone", + "smartphone" + ], + "phone": [ + "android", + "iphone", + "smartphone" + ] + }, + "distinctAttribute": null +} diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-6.snap b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-6.snap new file mode 100644 index 000000000..282cd6ba7 --- /dev/null +++ b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-6.snap @@ -0,0 +1,37 @@ +--- +source: dump/src/reader/mod.rs +expression: products.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness" + ], + "stopWords": [], + "synonyms": { + "android": [ + "phone", + "smartphone" + ], + "iphone": [ + "phone", + "smartphone" + ], + "phone": [ + "android", + "iphone", + "smartphone" + ] + }, + "distinctAttribute": null +} diff --git a/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-8.snap b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-8.snap new file mode 100644 index 000000000..d20fdc77e --- /dev/null +++ b/dump/src/reader/snapshots/dump__reader__test__import_dump_v1-8.snap @@ -0,0 +1,27 @@ +--- +source: dump/src/reader/mod.rs +expression: movies.settings().unwrap() +--- +{ + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [ + "genres", + "id" + ], + "rankingRules": [ + "typo", + "words", + "proximity", + "attribute", + "exactness", + "release_date:asc" + ], + "stopWords": [], + "synonyms": {}, + "distinctAttribute": null +} From b727fe7179ef361938244793f840b25f568df8c4 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 1 Dec 2022 10:55:17 +0100 Subject: [PATCH 13/17] Fix integration tests --- meilisearch-http/tests/dumps/mod.rs | 190 ++++++++++++++++++++++++++-- 1 file changed, 180 insertions(+), 10 deletions(-) diff --git a/meilisearch-http/tests/dumps/mod.rs b/meilisearch-http/tests/dumps/mod.rs index cd9ba3828..10098c60f 100644 --- a/meilisearch-http/tests/dumps/mod.rs +++ b/meilisearch-http/tests/dumps/mod.rs @@ -9,19 +9,189 @@ use crate::common::{default_settings, GetAllDocumentsOptions, Server}; // all the following test are ignored on windows. See #2364 #[actix_rt::test] #[cfg_attr(target_os = "windows", ignore)] -async fn import_dump_v1() { +async fn import_dump_v1_movie_raw() { + let temp = tempfile::tempdir().unwrap(); + let path = GetDump::MoviesRawV1.path(); + let options = Opt { import_dump: Some(path), ..default_settings(temp.path()) }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("indexUID")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("indexUID"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"genres": 53, "id": 53, "overview": 53, "poster": 53, "release_date": 53, "title": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({"displayedAttributes": ["*"], "searchableAttributes": ["*"], "filterableAttributes": [], "sortableAttributes": [], "rankingRules": ["typo", "words", "proximity", "attribute", "exactness"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks, + json!({ "results": [{"uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.317060500S", "enqueuedAt": "2021-09-08T09:08:45.153219Z", "startedAt": "2021-09-08T09:08:45.3961665Z", "finishedAt": "2021-09-08T09:08:54.713227Z" }], "limit": 20, "from": 0, "next": null }) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(100, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({"id": 100, "title": "Lock, Stock and Two Smoking Barrels", "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "genres": ["Comedy", "Crime"], "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000}) + ); + + let (document, code) = index.get_document(500, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({"id": 500, "title": "Reservoir Dogs", "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "genres": ["Crime", "Thriller"], "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + ); + + let (document, code) = index.get_document(10006, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({"id": 10006, "title": "Wild Seven", "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "genres": ["Action", "Crime", "Drama"], "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v1_movie_with_settings() { let temp = tempfile::tempdir().unwrap(); - for path in [ - GetDump::MoviesRawV1.path(), - GetDump::MoviesWithSettingsV1.path(), - GetDump::RubyGemsWithSettingsV1.path(), - ] { - let options = Opt { import_dump: Some(path), ..default_settings(temp.path()) }; - let error = Server::new_with_options(options).await.map(drop).unwrap_err(); + let path = GetDump::MoviesWithSettingsV1.path(); - assert_eq!(error.to_string(), "The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards."); - } + let options = Opt { import_dump: Some(path), ..default_settings(temp.path()) }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("indexUID")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("indexUID"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"genres": 53, "id": 53, "overview": 53, "poster": 53, "release_date": 53, "title": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({ "displayedAttributes": ["genres", "id", "overview", "poster", "release_date", "title"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "sortableAttributes": [], "rankingRules": ["typo", "words", "proximity", "attribute", "exactness"], "stopWords": ["of", "the"], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": { "oneTypo": 5, "twoTypos": 9 }, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 } }) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks, + json!({ "results": [{ "uid": 1, "indexUid": "indexUID", "status": "succeeded", "type": "settingsUpdate", "canceledBy": null, "details": { "displayedAttributes": ["genres", "id", "overview", "poster", "release_date", "title"], "searchableAttributes": ["title", "overview"], "filterableAttributes": ["genres"], "stopWords": ["of", "the"] }, "error": null, "duration": "PT7.288826907S", "enqueuedAt": "2021-09-08T09:34:40.882977Z", "startedAt": "2021-09-08T09:34:40.883073093Z", "finishedAt": "2021-09-08T09:34:48.1719Z"}, { "uid": 0, "indexUid": "indexUID", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": { "receivedDocuments": 0, "indexedDocuments": 31968 }, "error": null, "duration": "PT9.090735774S", "enqueuedAt": "2021-09-08T09:34:16.036101Z", "startedAt": "2021-09-08T09:34:16.261191226Z", "finishedAt": "2021-09-08T09:34:25.351927Z" }], "limit": 20, "from": 1, "next": null }) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(100, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 100, "title": "Lock, Stock and Two Smoking Barrels", "genres": ["Comedy", "Crime"], "overview": "A card shark and his unwillingly-enlisted friends need to make a lot of cash quick after losing a sketchy poker match. To do this they decide to pull a heist on a small-time gang who happen to be operating out of the flat next door.", "poster": "https://image.tmdb.org/t/p/w500/8kSerJrhrJWKLk1LViesGcnrUPE.jpg", "release_date": 889056000 }) + ); + + let (document, code) = index.get_document(500, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 500, "title": "Reservoir Dogs", "genres": ["Crime", "Thriller"], "overview": "A botched robbery indicates a police informant, and the pressure mounts in the aftermath at a warehouse. Crime begets violence as the survivors -- veteran Mr. White, newcomer Mr. Orange, psychopathic parolee Mr. Blonde, bickering weasel Mr. Pink and Nice Guy Eddie -- unravel.", "poster": "https://image.tmdb.org/t/p/w500/AjTtJNumZyUDz33VtMlF1K8JPsE.jpg", "release_date": 715392000}) + ); + + let (document, code) = index.get_document(10006, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "id": 10006, "title": "Wild Seven", "genres": ["Action", "Crime", "Drama"], "overview": "In this darkly karmic vision of Arizona, a man who breathes nothing but ill will begins a noxious domino effect as quickly as an uncontrollable virus kills. As he exits Arizona State Penn after twenty-one long years, Wilson has only one thing on the brain, leveling the score with career criminal, Mackey Willis.", "poster": "https://image.tmdb.org/t/p/w500/y114dTPoqn8k2Txps4P2tI95YCS.jpg", "release_date": 1136073600}) + ); +} + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn import_dump_v1_rubygems_with_settings() { + let temp = tempfile::tempdir().unwrap(); + + let path = GetDump::RubyGemsWithSettingsV1.path(); + + let options = Opt { import_dump: Some(path), ..default_settings(temp.path()) }; + let server = Server::new_with_options(options).await.unwrap(); + + let (indexes, code) = server.list_indexes(None, None).await; + assert_eq!(code, 200); + + assert_eq!(indexes["results"].as_array().unwrap().len(), 1); + assert_eq!(indexes["results"][0]["uid"], json!("rubygems")); + assert_eq!(indexes["results"][0]["primaryKey"], json!("id")); + + let index = server.index("rubygems"); + + let (stats, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!( + stats, + json!({ "numberOfDocuments": 53, "isIndexing": false, "fieldDistribution": {"description": 53, "id": 53, "name": 53, "summary": 53, "total_downloads": 53, "version": 53 }}) + ); + + let (settings, code) = index.settings().await; + assert_eq!(code, 200); + assert_eq!( + settings, + json!({"displayedAttributes": ["description", "id", "name", "summary", "total_downloads", "version"], "searchableAttributes": ["name", "summary"], "filterableAttributes": ["version"], "sortableAttributes": [], "rankingRules": ["typo", "words", "fame:desc", "proximity", "attribute", "exactness", "total_downloads:desc"], "stopWords": [], "synonyms": {}, "distinctAttribute": null, "typoTolerance": {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [] }, "faceting": { "maxValuesPerFacet": 100 }, "pagination": { "maxTotalHits": 1000 }}) + ); + + let (tasks, code) = index.list_tasks().await; + assert_eq!(code, 200); + assert_eq!( + tasks["results"][0], + json!({"uid": 92, "indexUid": "rubygems", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, "details": {"receivedDocuments": 0, "indexedDocuments": 1042}, "error": null, "duration": "PT1.487793839S", "enqueuedAt": "2021-09-08T09:27:01.465296Z", "startedAt": "2021-09-08T09:28:44.882177161Z", "finishedAt": "2021-09-08T09:28:46.369971Z"}) + ); + + // finally we're just going to check that we can still get a few documents by id + let (document, code) = index.get_document(188040, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "meilisearch", "summary": "An easy-to-use ruby client for Meilisearch API", "description": "An easy-to-use ruby client for Meilisearch API. See https://github.com/meilisearch/MeiliSearch", "id": "188040", "version": "0.15.2", "total_downloads": "7465"}) + ); + + let (document, code) = index.get_document(191940, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "doggo", "summary": "RSpec 3 formatter - documentation, with progress indication", "description": "Similar to \"rspec -f d\", but also indicates progress by showing the current test number and total test count on each line.", "id": "191940", "version": "1.1.0", "total_downloads": "9394"}) + ); + + let (document, code) = index.get_document(159227, None).await; + assert_eq!(code, 200); + assert_eq!( + document, + json!({ "name": "vortex-of-agony", "summary": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "description": "You dont need to use nodejs or go, just install this plugin. It will crash your application at random", "id": "159227", "version": "0.1.0", "total_downloads": "1007"}) + ); } #[actix_rt::test] From f892d122de2f6d41c7228cc25be270fa0b689041 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 17:02:14 +0000 Subject: [PATCH 14/17] Bump svenstaro/upload-release-action from 1.pre.release to 2.3.0 Bumps [svenstaro/upload-release-action](https://github.com/svenstaro/upload-release-action) from 1.pre.release to 2.3.0. - [Release notes](https://github.com/svenstaro/upload-release-action/releases) - [Changelog](https://github.com/svenstaro/upload-release-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/svenstaro/upload-release-action/compare/v1-release...2.3.0) --- updated-dependencies: - dependency-name: svenstaro/upload-release-action dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- .github/workflows/publish-binaries.yml | 6 +++--- .github/workflows/publish-deb-brew-pkg.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 9d06c35bc..db17d9d13 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -61,7 +61,7 @@ jobs: # No need to upload binaries for dry run (cron) - name: Upload binaries to release if: github.event_name == 'release' - uses: svenstaro/upload-release-action@v1-release + uses: svenstaro/upload-release-action@2.3.0 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/release/${{ matrix.artifact_name }} @@ -99,7 +99,7 @@ jobs: - name: Upload the binary to release # No need to upload binaries for dry run (cron) if: github.event_name == 'release' - uses: svenstaro/upload-release-action@v1-release + uses: svenstaro/upload-release-action@2.3.0 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/${{ matrix.target }}/release/meilisearch @@ -168,7 +168,7 @@ jobs: - name: Upload the binary to release # No need to upload binaries for dry run (cron) if: github.event_name == 'release' - uses: svenstaro/upload-release-action@v1-release + uses: svenstaro/upload-release-action@2.3.0 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/${{ matrix.target }}/release/meilisearch diff --git a/.github/workflows/publish-deb-brew-pkg.yml b/.github/workflows/publish-deb-brew-pkg.yml index 028001dcd..f4bf2d465 100644 --- a/.github/workflows/publish-deb-brew-pkg.yml +++ b/.github/workflows/publish-deb-brew-pkg.yml @@ -27,7 +27,7 @@ jobs: - name: Build deb package run: cargo deb -p meilisearch-http -o target/debian/meilisearch.deb - name: Upload debian pkg to release - uses: svenstaro/upload-release-action@v1-release + uses: svenstaro/upload-release-action@2.3.0 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/debian/meilisearch.deb From fc905f29e9264fde1624cf01d16ab3f78daeefcd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 17:02:18 +0000 Subject: [PATCH 15/17] Bump mislav/bump-homebrew-formula-action from 1 to 2 Bumps [mislav/bump-homebrew-formula-action](https://github.com/mislav/bump-homebrew-formula-action) from 1 to 2. - [Release notes](https://github.com/mislav/bump-homebrew-formula-action/releases) - [Commits](https://github.com/mislav/bump-homebrew-formula-action/compare/v1...v2) --- updated-dependencies: - dependency-name: mislav/bump-homebrew-formula-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/publish-deb-brew-pkg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-deb-brew-pkg.yml b/.github/workflows/publish-deb-brew-pkg.yml index 028001dcd..1677b1e5f 100644 --- a/.github/workflows/publish-deb-brew-pkg.yml +++ b/.github/workflows/publish-deb-brew-pkg.yml @@ -42,7 +42,7 @@ jobs: needs: check-version steps: - name: Create PR to Homebrew - uses: mislav/bump-homebrew-formula-action@v1 + uses: mislav/bump-homebrew-formula-action@v2 with: formula-name: meilisearch env: From dd6593f7b63dc184af572f69056695da6f60b5ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 17:02:25 +0000 Subject: [PATCH 16/17] Bump Swatinem/rust-cache from 2.0.0 to 2.2.0 Bumps [Swatinem/rust-cache](https://github.com/Swatinem/rust-cache) from 2.0.0 to 2.2.0. - [Release notes](https://github.com/Swatinem/rust-cache/releases) - [Changelog](https://github.com/Swatinem/rust-cache/blob/master/CHANGELOG.md) - [Commits](https://github.com/Swatinem/rust-cache/compare/v2.0.0...v2.2.0) --- updated-dependencies: - dependency-name: Swatinem/rust-cache dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/rust.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 266e306d6..8cde7d527 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Cache dependencies - uses: Swatinem/rust-cache@v2.0.0 + uses: Swatinem/rust-cache@v2.2.0 - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: @@ -49,7 +49,7 @@ jobs: toolchain: stable override: true - name: Cache dependencies - uses: Swatinem/rust-cache@v2.0.0 + uses: Swatinem/rust-cache@v2.2.0 - name: Run tests in debug uses: actions-rs/cargo@v1 with: @@ -68,7 +68,7 @@ jobs: override: true components: clippy - name: Cache dependencies - uses: Swatinem/rust-cache@v2.0.0 + uses: Swatinem/rust-cache@v2.2.0 - name: Run cargo clippy uses: actions-rs/cargo@v1 with: @@ -87,6 +87,6 @@ jobs: override: true components: rustfmt - name: Cache dependencies - uses: Swatinem/rust-cache@v2.0.0 + uses: Swatinem/rust-cache@v2.2.0 - name: Run cargo fmt run: cargo fmt --all -- --check From 7f3653ec3135aa08cafb09f148948f8c41809022 Mon Sep 17 00:00:00 2001 From: curquiza Date: Thu, 1 Dec 2022 18:23:29 +0100 Subject: [PATCH 17/17] Use ubuntu-latest when not impacting --- .github/workflows/coverage.yml | 2 +- .github/workflows/publish-deb-brew-pkg.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 3a10a611f..54d3b38b4 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -6,7 +6,7 @@ name: Execute code coverage jobs: nightly-coverage: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 diff --git a/.github/workflows/publish-deb-brew-pkg.yml b/.github/workflows/publish-deb-brew-pkg.yml index 028001dcd..c1e134d43 100644 --- a/.github/workflows/publish-deb-brew-pkg.yml +++ b/.github/workflows/publish-deb-brew-pkg.yml @@ -1,4 +1,4 @@ -name: Publish deb pkg to GitHub release & APT repository & Homebrew +name: Publish to APT repository & Homebrew on: release: @@ -38,7 +38,7 @@ jobs: homebrew: name: Bump Homebrew formula - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest needs: check-version steps: - name: Create PR to Homebrew