Merge #212 #222 #223

212: Introduce integration test on criteria r=Kerollmops a=ManyTheFish

- add pre-ranked dataset
- test each criterion 1 by 1
- test all criteria in several order

222: Move the `UpdateStore` into the http-ui crate r=Kerollmops a=Kerollmops

We no more need to have the `UpdateStore` inside of the mill crate as this is the job of the caller to stack the updates and sequentially give them to milli.

223: Update dataset links r=Kerollmops a=curquiza



Co-authored-by: many <maxime@meilisearch.com>
Co-authored-by: Many <legendre.maxime.isn@gmail.com>
Co-authored-by: Kerollmops <clement@meilisearch.com>
Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

Cargo.lock

@@ -961,6 +961,7 @@ dependencies = [
  "askama_warp",
  "byte-unit",
  "bytes 0.5.6",
+ "crossbeam-channel",
  "either",
  "flate2",
  "fst",
@@ -1365,7 +1366,6 @@ dependencies = [
  "bstr",
  "byteorder",
  "chrono",
- "crossbeam-channel",
  "csv",
  "either",
  "flate2",

benchmarks/README.md

@@ -87,7 +87,7 @@ The benchmarks are available for the following datasets:
 
 ### Songs
 
-`songs` is a subset of the [`songs.csv` dataset](https://meili-datasets.s3.fr-par.scw.cloud/songs.csv.gz).
+`songs` is a subset of the [`songs.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/songs.csv.gz).
 
 It was generated with this command:
 
@@ -95,11 +95,11 @@ It was generated with this command:
 xsv sample --seed 42 1000000 songs.csv -o smol-songs.csv
 ```
 
-_[Download the generated `songs` dataset](https://meili-datasets.s3.fr-par.scw.cloud/benchmarks/smol-songs.csv.gz)._
+_[Download the generated `songs` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-songs.csv.gz)._
 
 ### Wiki
 
-`wiki` is a subset of the [`wikipedia-articles.csv` dataset](https://meili-datasets.s3.fr-par.scw.cloud/wiki-articles.csv.gz).
+`wiki` is a subset of the [`wikipedia-articles.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/wiki-articles.csv.gz).
 
 It was generated with the following command:
 
@@ -107,5 +107,5 @@ It was generated with the following command:
 xsv sample --seed 42 500000 wiki-articles.csv -o smol-wiki-articles.csv
 ```
 
-_[Download the generated `wiki` dataset](https://meili-datasets.s3.fr-par.scw.cloud/benchmarks/smol-wiki-articles.csv.gz)._
+_[Download the generated `wiki` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-wiki-articles.csv.gz)._
 

benchmarks/build.rs

@@ -10,7 +10,7 @@ use convert_case::{Case, Casing};
 use flate2::read::GzDecoder;
 use reqwest::IntoUrl;
 
-const BASE_URL: &str = "https://meili-datasets.s3.fr-par.scw.cloud/benchmarks";
+const BASE_URL: &str = "https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets";
 
 const DATASET_SONGS: &str = "smol-songs";
 const DATASET_WIKI: &str = "smol-wiki-articles";

http-ui/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0.38"
 byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
+crossbeam-channel = "0.5.0"
 grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
 heed = "0.10.6"
 meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" }

http-ui/src/main.rs

@@ -1,3 +1,5 @@
+mod update_store;
+
 use std::{io, mem};
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::fmt::Display;
@@ -29,10 +31,12 @@ use tokio::sync::broadcast;
 use warp::{Filter, http::Response};
 use warp::filters::ws::Message;
 
-use milli::{FilterCondition, Index, MatchingWords, obkv_to_json, SearchResult, UpdateStore};
+use milli::{FilterCondition, Index, MatchingWords, obkv_to_json, SearchResult};
 use milli::update::{IndexDocumentsMethod, Setting, UpdateBuilder, UpdateFormat};
 use milli::update::UpdateIndexingStep::*;
 
+use self::update_store::UpdateStore;
+
 static GLOBAL_THREAD_POOL: OnceCell<ThreadPool> = OnceCell::new();
 
 #[derive(Debug, StructOpt)]

http-ui/src/update_store.rs

@@ -1,3 +1,5 @@
+#![allow(unused)]
+
 use std::path::Path;
 use std::sync::Arc;
 
@@ -6,7 +8,7 @@ use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice};
 use heed::{EnvOpenOptions, Env, Database};
 use serde::{Serialize, Deserialize};
 
-use crate::BEU64;
+pub type BEU64 = heed::zerocopy::U64<heed::byteorder::BE>;
 
 #[derive(Clone)]
 pub struct UpdateStore<M, N> {

milli/Cargo.toml

@@ -9,7 +9,6 @@ anyhow = "1.0.38"
 bstr = "0.2.15"
 byteorder = "1.4.2"
 chrono = { version = "0.4.19", features = ["serde"] }
-crossbeam-channel = "0.5.0"
 csv = "1.1.5"
 either = "1.6.1"
 flate2 = "1.0.20"

milli/src/lib.rs

@@ -4,7 +4,6 @@ mod criterion;
 mod external_documents_ids;
 mod fields_ids_map;
 mod search;
-mod update_store;
 pub mod facet;
 pub mod heed_codec;
 pub mod index;
@@ -29,7 +28,6 @@ pub use self::heed_codec::{RoaringBitmapLenCodec, BoRoaringBitmapLenCodec, CboRo
 pub use self::index::Index;
 pub use self::search::{Search, FacetDistribution, FilterCondition, SearchResult, MatchingWords};
 pub use self::tree_level::TreeLevel;
-pub use self::update_store::UpdateStore;
 
 pub type FastMap4<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher32>>;
 pub type FastMap8<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher64>>;

milli/tests/assets/test_set.ndjson

@@ -0,0 +1,17 @@
+{"id":"A","word_rank":0,"typo_rank":1,"proximity_rank":15,"attribute_rank":505,"exact_rank":5,"asc_desc_rank":0,"title":"hell o","description":"hell o is the fourteenth episode of the american television series glee performing songs with this word","tag":"blue","":""}
+{"id":"B","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":1,"title":"hello","description":"hello is a song recorded by english singer songwriter adele","tag":"red","":""}
+{"id":"C","word_rank":0,"typo_rank":1,"proximity_rank":8,"attribute_rank":336,"exact_rank":4,"asc_desc_rank":2,"title":"hell on earth","description":"hell on earth is the third studio album by american hip hop duo mobb deep","tag":"blue","":""}
+{"id":"D","word_rank":0,"typo_rank":1,"proximity_rank":10,"attribute_rank":757,"exact_rank":4,"asc_desc_rank":3,"title":"hell on wheels tv series","description":"the construction of the first transcontinental railroad across the united states in the world","tag":"red","":""}
+{"id":"E","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":1,"asc_desc_rank":4,"title":"hello kitty","description":"also known by her full name kitty white is a fictional character produced by the japanese company sanrio","tag":"green","":""}
+{"id":"F","word_rank":2,"typo_rank":1,"proximity_rank":0,"attribute_rank":1017,"exact_rank":5,"asc_desc_rank":5,"title":"laptop orchestra","description":"a laptop orchestra lork or lo is a chamber music ensemble consisting primarily of laptops like helo huddersfield experimental laptop orchestra","tag":"blue","":""}
+{"id":"G","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":5,"title":"hello world film","description":"hello world is a 2019 japanese animated sci fi romantic drama film directed by tomohiko ito and produced by graphinica","tag":"red","":""}
+{"id":"H","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":4,"title":"world hello day","description":"holiday observed on november 21 to express that conflicts should be resolved through communication rather than the use of force","tag":"green","":""}
+{"id":"I","word_rank":0,"typo_rank":0,"proximity_rank":8,"attribute_rank":338,"exact_rank":3,"asc_desc_rank":3,"title":"hello world song","description":"hello world is a song written by tom douglas tony lane and david lee and recorded by american country music group lady antebellum","tag":"blue","":""}
+{"id":"J","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":1,"exact_rank":3,"asc_desc_rank":2,"title":"hello cruel world","description":"hello cruel world is an album by new zealand band tall dwarfs","tag":"green","":""}
+{"id":"K","word_rank":0,"typo_rank":2,"proximity_rank":9,"attribute_rank":670,"exact_rank":5,"asc_desc_rank":1,"title":"ello creation system","description":"in few word ello was a construction toy created by the american company mattel to engage girls in construction play","tag":"red","":""}
+{"id":"L","word_rank":0,"typo_rank":0,"proximity_rank":2,"attribute_rank":250,"exact_rank":4,"asc_desc_rank":0,"title":"good morning world","description":"good morning world is an american sitcom broadcast on cbs tv during the 1967 1968 season","tag":"blue","":""}
+{"id":"M","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":0,"asc_desc_rank":0,"title":"hello world america","description":"a perfect match for a perfect engine using the query hello world america","tag":"red","":""}
+{"id":"N","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":1,"asc_desc_rank":4,"title":"hello world america unleashed","description":"a very good match for a very good engine using the query hello world america","tag":"green","":""}
+{"id":"O","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":10,"exact_rank":0,"asc_desc_rank":6,"title":"a perfect match for a perfect engine using the query hello world america","description":"hello world america","tag":"blue","":""}
+{"id":"P","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":12,"exact_rank":1,"asc_desc_rank":3,"title":"a very good match for a very good engine using the query hello world america","description":"hello world america unleashed","tag":"red","":""}
+{"id":"Q","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":2,"title":"hello world","description":"a hello world program generally is a computer program that outputs or displays the message hello world","tag":"green","":""}

milli/tests/mod.rs

@@ -0,0 +1 @@
+mod search;

milli/tests/search/mod.rs

@@ -0,0 +1,125 @@
+use milli::{Criterion, Index, DocumentId};
+use milli::update::{IndexDocuments, UpdateFormat, Settings};
+
+use big_s::S;
+use heed::EnvOpenOptions;
+use maplit::{hashmap, hashset};
+use serde::Deserialize;
+use slice_group_by::GroupBy;
+
+mod query_criteria;
+
+pub const TEST_QUERY: &'static str = "hello world america";
+
+pub const EXTERNAL_DOCUMENTS_IDS: &[&str; 17] = &["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q"];
+
+pub const CONTENT: &str = include_str!("../assets/test_set.ndjson");
+
+pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
+    let path = tempfile::tempdir().unwrap();
+    let mut options = EnvOpenOptions::new();
+    options.map_size(10 * 1024 * 1024); // 10 MB
+    let index = Index::new(options, &path).unwrap();
+
+    let mut wtxn = index.write_txn().unwrap();
+
+    let mut builder = Settings::new(&mut wtxn, &index, 0);
+
+    let criteria = criteria.iter().map(|c| c.to_string()).collect();
+    builder.set_criteria(criteria);
+    builder.set_filterable_fields(hashset!{
+        S("tag"),
+        S("asc_desc_rank"),
+    });
+    builder.set_synonyms(hashmap!{
+        S("hello") => vec![S("good morning")],
+        S("world") => vec![S("earth")],
+        S("america") => vec![S("the united states")],
+    });
+    builder.set_searchable_fields(vec![S("title"),S("description")]);
+    builder.execute(|_, _| ()).unwrap();
+
+    // index documents
+    let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
+    builder.update_format(UpdateFormat::JsonStream);
+    builder.enable_autogenerate_docids();
+    builder.execute(CONTENT.as_bytes(), |_, _| ()).unwrap();
+
+    wtxn.commit().unwrap();
+
+    index
+}
+
+pub fn internal_to_external_ids(index: &Index, internal_ids: &[DocumentId]) -> Vec<String> {
+    let mut rtxn = index.read_txn().unwrap();
+    let docid_map = index.external_documents_ids(&mut rtxn).unwrap();
+    let docid_map: std::collections::HashMap<_, _> = EXTERNAL_DOCUMENTS_IDS.iter().map(|id| (docid_map.get(id).unwrap(), id)).collect();
+    internal_ids.iter().map(|id| docid_map.get(id).unwrap().to_string()).collect()
+}
+
+pub fn expected_order(criteria: &[Criterion], authorize_typo: bool, optional_words: bool) -> Vec<TestDocument> {
+    let dataset = serde_json::Deserializer::from_str(CONTENT).into_iter().map(|r| r.unwrap()).collect();
+    let mut groups: Vec<Vec<TestDocument>> = vec![dataset];
+
+    for criterion in criteria {
+        let mut new_groups = Vec::new();
+        for group in groups.iter_mut() {
+            match criterion {
+                Criterion::Attribute => {
+                    group.sort_by_key(|d| d.attribute_rank);
+                    new_groups.extend(group.linear_group_by_key(|d| d.attribute_rank).map(Vec::from));
+                },
+                Criterion::Exactness => {
+                    group.sort_by_key(|d| d.exact_rank);
+                    new_groups.extend(group.linear_group_by_key(|d| d.exact_rank).map(Vec::from));
+                },
+                Criterion::Proximity => {
+                    group.sort_by_key(|d| d.proximity_rank);
+                    new_groups.extend(group.linear_group_by_key(|d| d.proximity_rank).map(Vec::from));
+                },
+                Criterion::Typo => {
+                    group.sort_by_key(|d| d.typo_rank);
+                    new_groups.extend(group.linear_group_by_key(|d| d.typo_rank).map(Vec::from));
+                },
+                Criterion::Words => {
+                    group.sort_by_key(|d| d.word_rank);
+                    new_groups.extend(group.linear_group_by_key(|d| d.word_rank).map(Vec::from));
+                },
+                Criterion::Asc(field_name) if field_name == "asc_desc_rank" => {
+                    group.sort_by_key(|d| d.asc_desc_rank);
+                    new_groups.extend(group.linear_group_by_key(|d| d.asc_desc_rank).map(Vec::from));
+                },
+                Criterion::Desc(field_name)  if field_name == "asc_desc_rank" => {
+                    group.sort_by_key(|d| std::cmp::Reverse(d.asc_desc_rank));
+                    new_groups.extend(group.linear_group_by_key(|d| d.asc_desc_rank).map(Vec::from));
+                },
+                Criterion::Asc(_) | Criterion::Desc(_) => new_groups.push(group.clone()),
+            }
+        }
+        groups = std::mem::take(&mut new_groups);
+    }
+
+    if authorize_typo && optional_words {
+        groups.into_iter().flatten().collect()
+    } else if optional_words {
+        groups.into_iter().flatten().filter(|d| d.typo_rank == 0).collect()
+    } else if authorize_typo {
+        groups.into_iter().flatten().filter(|d| d.word_rank == 0).collect()
+    } else {
+        groups.into_iter().flatten().filter(|d| d.word_rank == 0 && d.typo_rank == 0).collect()
+    }
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct TestDocument {
+    pub id: String,
+    pub word_rank: u32,
+    pub typo_rank: u32,
+    pub proximity_rank: u32,
+    pub attribute_rank: u32,
+    pub exact_rank: u32,
+    pub asc_desc_rank: u32,
+    pub title: String,
+    pub description: String,
+    pub tag: String,
+}

milli/tests/search/query_criteria.rs

@@ -0,0 +1,213 @@
+use milli::{Search, SearchResult, Criterion};
+use big_s::S;
+
+use crate::search::{self, EXTERNAL_DOCUMENTS_IDS};
+use Criterion::*;
+
+const ALLOW_TYPOS: bool = true;
+const DISALLOW_TYPOS: bool = false;
+const ALLOW_OPTIONAL_WORDS: bool = true;
+const DISALLOW_OPTIONAL_WORDS: bool = false;
+
+macro_rules! test_criterion {
+    ($func:ident, $optional_word:ident, $authorize_typos:ident $(, $criterion:expr)?) => {
+        #[test]
+        fn $func() {
+            let criteria = vec![$($criterion)?];
+            let index = search::setup_search_index_with_criteria(&criteria);
+            let mut rtxn = index.read_txn().unwrap();
+
+            let mut search = Search::new(&mut rtxn, &index);
+            search.query(search::TEST_QUERY);
+            search.limit(EXTERNAL_DOCUMENTS_IDS.len());
+            search.authorize_typos($authorize_typos);
+            search.optional_words($optional_word);
+
+            let SearchResult { documents_ids, .. } = search.execute().unwrap();
+
+            let expected_external_ids: Vec<_> = search::expected_order(&criteria, $authorize_typos, $optional_word)
+                .into_iter()
+                .map(|d| d.id).collect();
+            let documents_ids = search::internal_to_external_ids(&index, &documents_ids);
+            assert_eq!(documents_ids, expected_external_ids);
+        }
+    }
+}
+
+test_criterion!(none_allow_typo,                     ALLOW_OPTIONAL_WORDS,      ALLOW_TYPOS);
+test_criterion!(none_disallow_typo,                  DISALLOW_OPTIONAL_WORDS,   DISALLOW_TYPOS);
+test_criterion!(words_allow_typo,                    ALLOW_OPTIONAL_WORDS,      ALLOW_TYPOS,    Words);
+test_criterion!(attribute_allow_typo,                DISALLOW_OPTIONAL_WORDS,   ALLOW_TYPOS,    Attribute);
+test_criterion!(attribute_disallow_typo,             DISALLOW_OPTIONAL_WORDS,   DISALLOW_TYPOS, Attribute);
+test_criterion!(exactness_allow_typo,                DISALLOW_OPTIONAL_WORDS,   ALLOW_TYPOS,    Exactness);
+test_criterion!(exactness_disallow_typo,             DISALLOW_OPTIONAL_WORDS,   DISALLOW_TYPOS, Exactness);
+test_criterion!(proximity_allow_typo,                DISALLOW_OPTIONAL_WORDS,   ALLOW_TYPOS,    Proximity);
+test_criterion!(proximity_disallow_typo,             DISALLOW_OPTIONAL_WORDS,   DISALLOW_TYPOS, Proximity);
+test_criterion!(asc_allow_typo,                      DISALLOW_OPTIONAL_WORDS,   ALLOW_TYPOS,    Asc(S("asc_desc_rank")));
+test_criterion!(asc_disallow_typo,                   DISALLOW_OPTIONAL_WORDS,   DISALLOW_TYPOS, Asc(S("asc_desc_rank")));
+test_criterion!(desc_allow_typo,                     DISALLOW_OPTIONAL_WORDS,   ALLOW_TYPOS,    Desc(S("asc_desc_rank")));
+test_criterion!(desc_disallow_typo,                  DISALLOW_OPTIONAL_WORDS,   DISALLOW_TYPOS, Desc(S("asc_desc_rank")));
+test_criterion!(asc_unexisting_field_allow_typo,     DISALLOW_OPTIONAL_WORDS,   ALLOW_TYPOS,    Asc(S("unexisting_field")));
+test_criterion!(asc_unexisting_field_disallow_typo,  DISALLOW_OPTIONAL_WORDS,   DISALLOW_TYPOS, Asc(S("unexisting_field")));
+test_criterion!(desc_unexisting_field_allow_typo,    DISALLOW_OPTIONAL_WORDS,   ALLOW_TYPOS,    Desc(S("unexisting_field")));
+test_criterion!(desc_unexisting_field_disallow_typo, DISALLOW_OPTIONAL_WORDS,   DISALLOW_TYPOS, Desc(S("unexisting_field")));
+
+#[test]
+fn criteria_mixup() {
+    use Criterion::*;
+    let index = search::setup_search_index_with_criteria(&vec![Words, Attribute, Desc(S("asc_desc_rank")), Exactness, Proximity, Typo]);
+
+    let criteria_mix = {
+        // Criterion doesn't implement Copy, we create a new Criterion using a closure
+        let desc = || Desc(S("asc_desc_rank"));
+        // all possible criteria order
+        vec![
+            vec![Words, Attribute,  desc(),     Exactness,  Proximity,  Typo],
+            vec![Words, Attribute,  desc(),     Exactness,  Typo,       Proximity],
+            vec![Words, Attribute,  desc(),     Proximity,  Exactness,  Typo],
+            vec![Words, Attribute,  desc(),     Proximity,  Typo,       Exactness],
+            vec![Words, Attribute,  desc(),     Typo,       Exactness,  Proximity],
+            vec![Words, Attribute,  desc(),     Typo,       Proximity,  Exactness],
+            vec![Words, Attribute,  Exactness,  desc(),     Proximity,  Typo],
+            vec![Words, Attribute,  Exactness,  desc(),     Typo,       Proximity],
+            vec![Words, Attribute,  Exactness,  Proximity,  desc(),     Typo],
+            vec![Words, Attribute,  Exactness,  Proximity,  Typo,       desc()],
+            vec![Words, Attribute,  Exactness,  Typo,       desc(),     Proximity],
+            vec![Words, Attribute,  Exactness,  Typo,       Proximity,  desc()],
+            vec![Words, Attribute,  Proximity,  desc(),     Exactness,  Typo],
+            vec![Words, Attribute,  Proximity,  desc(),     Typo,       Exactness],
+            vec![Words, Attribute,  Proximity,  Exactness,  desc(),     Typo],
+            vec![Words, Attribute,  Proximity,  Exactness,  Typo,       desc()],
+            vec![Words, Attribute,  Proximity,  Typo,       desc(),     Exactness],
+            vec![Words, Attribute,  Proximity,  Typo,       Exactness,  desc()],
+            vec![Words, Attribute,  Typo,       desc(),     Exactness,  Proximity],
+            vec![Words, Attribute,  Typo,       desc(),     Proximity,  Exactness],
+            vec![Words, Attribute,  Typo,       Exactness,  desc(),     Proximity],
+            vec![Words, Attribute,  Typo,       Exactness,  Proximity,  desc()],
+            vec![Words, Attribute,  Typo,       Proximity,  desc(),     Exactness],
+            vec![Words, Attribute,  Typo,       Proximity,  Exactness,  desc()],
+            vec![Words, desc(),     Attribute,  Exactness,  Proximity,  Typo],
+            vec![Words, desc(),     Attribute,  Exactness,  Typo,       Proximity],
+            vec![Words, desc(),     Attribute,  Proximity,  Exactness,  Typo],
+            vec![Words, desc(),     Attribute,  Proximity,  Typo,       Exactness],
+            vec![Words, desc(),     Attribute,  Typo,       Exactness,  Proximity],
+            vec![Words, desc(),     Attribute,  Typo,       Proximity,  Exactness],
+            vec![Words, desc(),     Exactness,  Attribute,  Proximity,  Typo],
+            vec![Words, desc(),     Exactness,  Attribute,  Typo,       Proximity],
+            vec![Words, desc(),     Exactness,  Proximity,  Attribute,  Typo],
+            vec![Words, desc(),     Exactness,  Proximity,  Typo,       Attribute],
+            vec![Words, desc(),     Exactness,  Typo,       Attribute,  Proximity],
+            vec![Words, desc(),     Exactness,  Typo,       Proximity,  Attribute],
+            vec![Words, desc(),     Proximity,  Attribute,  Exactness,  Typo],
+            vec![Words, desc(),     Proximity,  Attribute,  Typo,       Exactness],
+            vec![Words, desc(),     Proximity,  Exactness,  Attribute,  Typo],
+            vec![Words, desc(),     Proximity,  Exactness,  Typo,       Attribute],
+            vec![Words, desc(),     Proximity,  Typo,       Attribute,  Exactness],
+            vec![Words, desc(),     Proximity,  Typo,       Exactness,  Attribute],
+            vec![Words, desc(),     Typo,       Attribute,  Exactness,  Proximity],
+            vec![Words, desc(),     Typo,       Attribute,  Proximity,  Exactness],
+            vec![Words, desc(),     Typo,       Exactness,  Attribute,  Proximity],
+            vec![Words, desc(),     Typo,       Exactness,  Proximity,  Attribute],
+            vec![Words, desc(),     Typo,       Proximity,  Attribute,  Exactness],
+            vec![Words, desc(),     Typo,       Proximity,  Exactness,  Attribute],
+            vec![Words, Exactness,  Attribute,  desc(),     Proximity,  Typo],
+            vec![Words, Exactness,  Attribute,  desc(),     Typo,       Proximity],
+            vec![Words, Exactness,  Attribute,  Proximity,  desc(),     Typo],
+            vec![Words, Exactness,  Attribute,  Proximity,  Typo,       desc()],
+            vec![Words, Exactness,  Attribute,  Typo,       desc(),     Proximity],
+            vec![Words, Exactness,  Attribute,  Typo,       Proximity,  desc()],
+            vec![Words, Exactness,  desc(),     Attribute,  Proximity,  Typo],
+            vec![Words, Exactness,  desc(),     Attribute,  Typo,       Proximity],
+            vec![Words, Exactness,  desc(),     Proximity,  Attribute,  Typo],
+            vec![Words, Exactness,  desc(),     Proximity,  Typo,       Attribute],
+            vec![Words, Exactness,  desc(),     Typo,       Attribute,  Proximity],
+            vec![Words, Exactness,  desc(),     Typo,       Proximity,  Attribute],
+            vec![Words, Exactness,  Proximity,  Attribute,  desc(),     Typo],
+            vec![Words, Exactness,  Proximity,  Attribute,  Typo,       desc()],
+            vec![Words, Exactness,  Proximity,  desc(),     Attribute,  Typo],
+            vec![Words, Exactness,  Proximity,  desc(),     Typo,       Attribute],
+            vec![Words, Exactness,  Proximity,  Typo,       Attribute,  desc()],
+            vec![Words, Exactness,  Proximity,  Typo,       desc(),     Attribute],
+            vec![Words, Exactness,  Typo,       Attribute,  desc(),     Proximity],
+            vec![Words, Exactness,  Typo,       Attribute,  Proximity,  desc()],
+            vec![Words, Exactness,  Typo,       desc(),     Attribute,  Proximity],
+            vec![Words, Exactness,  Typo,       desc(),     Proximity,  Attribute],
+            vec![Words, Exactness,  Typo,       Proximity,  Attribute,  desc()],
+            vec![Words, Exactness,  Typo,       Proximity,  desc(),     Attribute],
+            vec![Words, Proximity,  Attribute,  desc(),     Exactness,  Typo],
+            vec![Words, Proximity,  Attribute,  desc(),     Typo,       Exactness],
+            vec![Words, Proximity,  Attribute,  Exactness,  desc(),     Typo],
+            vec![Words, Proximity,  Attribute,  Exactness,  Typo,       desc()],
+            vec![Words, Proximity,  Attribute,  Typo,       desc(),     Exactness],
+            vec![Words, Proximity,  Attribute,  Typo,       Exactness,  desc()],
+            vec![Words, Proximity,  desc(),     Attribute,  Exactness,  Typo],
+            vec![Words, Proximity,  desc(),     Attribute,  Typo,       Exactness],
+            vec![Words, Proximity,  desc(),     Exactness,  Attribute,  Typo],
+            vec![Words, Proximity,  desc(),     Exactness,  Typo,       Attribute],
+            vec![Words, Proximity,  desc(),     Typo,       Attribute,  Exactness],
+            vec![Words, Proximity,  desc(),     Typo,       Exactness,  Attribute],
+            vec![Words, Proximity,  Exactness,  Attribute,  desc(),     Typo],
+            vec![Words, Proximity,  Exactness,  Attribute,  Typo,       desc()],
+            vec![Words, Proximity,  Exactness,  desc(),     Attribute,  Typo],
+            vec![Words, Proximity,  Exactness,  desc(),     Typo,       Attribute],
+            vec![Words, Proximity,  Exactness,  Typo,       Attribute,  desc()],
+            vec![Words, Proximity,  Exactness,  Typo,       desc(),     Attribute],
+            vec![Words, Proximity,  Typo,       Attribute,  desc(),     Exactness],
+            vec![Words, Proximity,  Typo,       Attribute,  Exactness,  desc()],
+            vec![Words, Proximity,  Typo,       desc(),     Attribute,  Exactness],
+            vec![Words, Proximity,  Typo,       desc(),     Exactness,  Attribute],
+            vec![Words, Proximity,  Typo,       Exactness,  Attribute,  desc()],
+            vec![Words, Proximity,  Typo,       Exactness,  desc(),     Attribute],
+            vec![Words, Typo,       Attribute,  desc(),     Exactness,  Proximity],
+            vec![Words, Typo,       Attribute,  desc(),     Proximity,  Exactness],
+            vec![Words, Typo,       Attribute,  Exactness,  desc(),     Proximity],
+            vec![Words, Typo,       Attribute,  Exactness,  Proximity,  desc()],
+            vec![Words, Typo,       Attribute,  Proximity,  desc(),     Exactness],
+            vec![Words, Typo,       Attribute,  Proximity,  Exactness,  desc()],
+            vec![Words, Typo,       desc(),     Attribute,  Proximity,  Exactness],
+            vec![Words, Typo,       desc(),     Exactness,  Attribute,  Proximity],
+            vec![Words, Typo,       desc(),     Exactness,  Attribute,  Proximity],
+            vec![Words, Typo,       desc(),     Exactness,  Proximity,  Attribute],
+            vec![Words, Typo,       desc(),     Proximity,  Attribute,  Exactness],
+            vec![Words, Typo,       desc(),     Proximity,  Exactness,  Attribute],
+            vec![Words, Typo,       Exactness,  Attribute,  desc(),     Proximity],
+            vec![Words, Typo,       Exactness,  Attribute,  Proximity,  desc()],
+            vec![Words, Typo,       Exactness,  desc(),     Attribute,  Proximity],
+            vec![Words, Typo,       Exactness,  desc(),     Proximity,  Attribute],
+            vec![Words, Typo,       Exactness,  Proximity,  Attribute,  desc()],
+            vec![Words, Typo,       Exactness,  Proximity,  desc(),     Attribute],
+            vec![Words, Typo,       Proximity,  Attribute,  desc(),     Exactness],
+            vec![Words, Typo,       Proximity,  Attribute,  Exactness,  desc()],
+            vec![Words, Typo,       Proximity,  desc(),     Attribute,  Exactness],
+            vec![Words, Typo,       Proximity,  desc(),     Exactness,  Attribute],
+            vec![Words, Typo,       Proximity,  Exactness,  Attribute,  desc()],
+            vec![Words, Typo,       Proximity,  Exactness,  desc(),     Attribute],
+        ]
+    };
+
+    for criteria in criteria_mix {
+        eprintln!("Testing with criteria order: {:?}", &criteria);
+        //update criteria
+        let mut wtxn = index.write_txn().unwrap();
+        index.put_criteria(&mut wtxn, &criteria).unwrap();
+        wtxn.commit().unwrap();
+
+        let mut rtxn = index.read_txn().unwrap();
+
+        let mut search = Search::new(&mut rtxn, &index);
+        search.query(search::TEST_QUERY);
+        search.limit(EXTERNAL_DOCUMENTS_IDS.len());
+        search.optional_words(ALLOW_OPTIONAL_WORDS);
+        search.authorize_typos(ALLOW_TYPOS);
+
+        let SearchResult { documents_ids, .. } = search.execute().unwrap();
+
+        let expected_external_ids: Vec<_> = search::expected_order(&criteria, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS)
+            .into_iter()
+            .map(|d| d.id)
+            .collect();
+        let documents_ids = search::internal_to_external_ids(&index, &documents_ids);
+
+        assert_eq!(documents_ids, expected_external_ids);
+    }
+}