From 10882bcbceb58b706f3bc7ae0b1818f48b02d1db Mon Sep 17 00:00:00 2001 From: many Date: Thu, 3 Jun 2021 14:44:53 +0200 Subject: [PATCH 1/8] Introduce integration test on criteria --- milli/tests/assets/test_set.ndjson | 17 + milli/tests/mod.rs | 1 + milli/tests/search/mod.rs | 137 +++++++ milli/tests/search/query_criteria.rs | 531 +++++++++++++++++++++++++++ 4 files changed, 686 insertions(+) create mode 100644 milli/tests/assets/test_set.ndjson create mode 100644 milli/tests/mod.rs create mode 100644 milli/tests/search/mod.rs create mode 100644 milli/tests/search/query_criteria.rs diff --git a/milli/tests/assets/test_set.ndjson b/milli/tests/assets/test_set.ndjson new file mode 100644 index 000000000..f219ab7e9 --- /dev/null +++ b/milli/tests/assets/test_set.ndjson @@ -0,0 +1,17 @@ +{"id":"A","word_rank":0,"typo_rank":1,"proximity_rank":15,"attribute_rank":505,"exact_rank":5,"asc_desc_rank":0,"title":"hell o","description":"hell o is the fourteenth episode of the american television series glee performing songs with this word","tag":"blue","":""} +{"id":"B","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":4,"asc_desc_rank":1,"title":"hello","description":"hello is a song recorded by english singer songwriter adele","tag":"red","":""} +{"id":"C","word_rank":0,"typo_rank":1,"proximity_rank":8,"attribute_rank":336,"exact_rank":4,"asc_desc_rank":2,"title":"hell on earth","description":"hell on earth is the third studio album by american hip hop duo mobb deep","tag":"blue","":""} +{"id":"D","word_rank":0,"typo_rank":1,"proximity_rank":10,"attribute_rank":757,"exact_rank":4,"asc_desc_rank":3,"title":"hell on wheels tv series","description":"the construction of the first transcontinental railroad across the united states in the world","tag":"red","":""} +{"id":"E","word_rank":2,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":1,"asc_desc_rank":4,"title":"hello kitty","description":"also known by her full name kitty white is a fictional character produced by the japanese company sanrio","tag":"green","":""} +{"id":"F","word_rank":2,"typo_rank":1,"proximity_rank":0,"attribute_rank":1017,"exact_rank":5,"asc_desc_rank":5,"title":"laptop orchestra","description":"a laptop orchestra lork or lo is a chamber music ensemble consisting primarily of laptops like helo huddersfield experimental laptop orchestra","tag":"blue","":""} +{"id":"G","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":5,"title":"hello world film","description":"hello world is a 2019 japanese animated sci fi romantic drama film directed by tomohiko ito and produced by graphinica","tag":"red","":""} +{"id":"H","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":4,"title":"world hello day","description":"holiday observed on november 21 to express that conflicts should be resolved through communication rather than the use of force","tag":"green","":""} +{"id":"I","word_rank":0,"typo_rank":0,"proximity_rank":8,"attribute_rank":338,"exact_rank":3,"asc_desc_rank":3,"title":"hello world song","description":"hello world is a song written by tom douglas tony lane and david lee and recorded by american country music group lady antebellum","tag":"blue","":""} +{"id":"J","word_rank":1,"typo_rank":0,"proximity_rank":1,"attribute_rank":1,"exact_rank":3,"asc_desc_rank":2,"title":"hello cruel world","description":"hello cruel world is an album by new zealand band tall dwarfs","tag":"green","":""} +{"id":"K","word_rank":0,"typo_rank":2,"proximity_rank":9,"attribute_rank":670,"exact_rank":5,"asc_desc_rank":1,"title":"ello creation system","description":"in few word ello was a construction toy created by the american company mattel to engage girls in construction play","tag":"red","":""} +{"id":"L","word_rank":0,"typo_rank":0,"proximity_rank":2,"attribute_rank":250,"exact_rank":4,"asc_desc_rank":0,"title":"good morning world","description":"good morning world is an american sitcom broadcast on cbs tv during the 1967 1968 season","tag":"blue","":""} +{"id":"M","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":0,"asc_desc_rank":0,"title":"hello world america","description":"a perfect match for a perfect engine using the query hello world america","tag":"red","":""} +{"id":"N","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":1,"asc_desc_rank":4,"title":"hello world america unleashed","description":"a very good match for a very good engine using the query hello world america","tag":"green","":""} +{"id":"O","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":10,"exact_rank":0,"asc_desc_rank":6,"title":"a perfect match for a perfect engine using the query hello world america","description":"hello world america","tag":"blue","":""} +{"id":"P","word_rank":0,"typo_rank":0,"proximity_rank":0,"attribute_rank":12,"exact_rank":1,"asc_desc_rank":3,"title":"a very good match for a very good engine using the query hello world america","description":"hello world america unleashed","tag":"red","":""} +{"id":"Q","word_rank":1,"typo_rank":0,"proximity_rank":0,"attribute_rank":0,"exact_rank":3,"asc_desc_rank":2,"title":"hello world","description":"a hello world program generally is a computer program that outputs or displays the message hello world","tag":"green","":""} diff --git a/milli/tests/mod.rs b/milli/tests/mod.rs new file mode 100644 index 000000000..11095a6a9 --- /dev/null +++ b/milli/tests/mod.rs @@ -0,0 +1 @@ +mod search; diff --git a/milli/tests/search/mod.rs b/milli/tests/search/mod.rs new file mode 100644 index 000000000..609787712 --- /dev/null +++ b/milli/tests/search/mod.rs @@ -0,0 +1,137 @@ +use milli::{Criterion, Index, DocumentId}; +use milli::update::{IndexDocuments, UpdateFormat, Settings}; + +use big_s::S; +use heed::EnvOpenOptions; +use maplit::{hashmap, hashset}; +use serde::Deserialize; +use slice_group_by::GroupBy; + +mod query_criteria; + +pub const TEST_QUERY: &'static str = "hello world america"; + +pub const EXTERNAL_DOCUMENTS_IDS: &[&str; 17] = &["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q"]; + +pub const CONTENT: &str = include_str!("../assets/test_set.ndjson"); + +pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { + let path = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(10 * 1024 * 1024); // 10 MB + let index = Index::new(options, &path).unwrap(); + + let mut wtxn = index.write_txn().unwrap(); + + let mut builder = Settings::new(&mut wtxn, &index, 0); + + let criteria = criteria.iter().map(|c| c.to_string()).collect(); + builder.set_criteria(criteria); + builder.set_filterable_fields(hashset!{ + S("tag"), + S("unexisting_field"), + S("asc_desc_rank"), + S("unexisting_field"), + }); + builder.set_synonyms(hashmap!{ + S("hello") => vec![S("good morning")], + S("world") => vec![S("earth")], + S("america") => vec![S("the united states")], + }); + builder.set_searchable_fields(vec![S("title"),S("description")]); + builder.execute(|_, _| ()).unwrap(); + + // index documents + let mut builder = IndexDocuments::new(&mut wtxn, &index, 0); + builder.update_format(UpdateFormat::JsonStream); + builder.enable_autogenerate_docids(); + builder.execute(CONTENT.as_bytes(), |_, _| ()).unwrap(); + + wtxn.commit().unwrap(); + + index +} + +#[allow(dead_code)] +pub fn external_to_internal_ids(index: &Index, external_ids: &[&str]) -> Vec { + let mut rtxn = index.read_txn().unwrap(); + let docid_map = index.external_documents_ids(&mut rtxn).unwrap(); + external_ids.iter().map(|id| docid_map.get(id).unwrap()).collect() +} + +pub fn internal_to_external_ids(index: &Index, internal_ids: &[DocumentId]) -> Vec { + let mut rtxn = index.read_txn().unwrap(); + let docid_map = index.external_documents_ids(&mut rtxn).unwrap(); + let docid_map: std::collections::HashMap<_, _> = EXTERNAL_DOCUMENTS_IDS.iter().map(|id| (docid_map.get(id).unwrap(), id)).collect(); + internal_ids.iter().map(|id| docid_map.get(id).unwrap().to_string()).collect() +} + +fn fetch_dataset() -> Vec { + serde_json::Deserializer::from_str(CONTENT).into_iter().map(|r| r.unwrap()).collect() +} + +pub fn expected_order(criteria: &[Criterion], autorize_typo: bool, optional_words: bool) -> Vec { + let dataset = fetch_dataset(); + let mut groups: Vec> = vec![dataset]; + + for criterion in criteria { + let mut new_groups = Vec::new(); + for group in groups.iter_mut() { + match criterion { + Criterion::Attribute => { + group.sort_by_key(|d| d.attribute_rank); + new_groups.extend(group.linear_group_by_key(|d| d.attribute_rank).map(Vec::from)); + }, + Criterion::Exactness => { + group.sort_by_key(|d| d.exact_rank); + new_groups.extend(group.linear_group_by_key(|d| d.exact_rank).map(Vec::from)); + }, + Criterion::Proximity => { + group.sort_by_key(|d| d.proximity_rank); + new_groups.extend(group.linear_group_by_key(|d| d.proximity_rank).map(Vec::from)); + }, + Criterion::Typo => { + group.sort_by_key(|d| d.typo_rank); + new_groups.extend(group.linear_group_by_key(|d| d.typo_rank).map(Vec::from)); + }, + Criterion::Words => { + group.sort_by_key(|d| d.word_rank); + new_groups.extend(group.linear_group_by_key(|d| d.word_rank).map(Vec::from)); + }, + Criterion::Asc(_) => { + group.sort_by_key(|d| d.asc_desc_rank); + new_groups.extend(group.linear_group_by_key(|d| d.asc_desc_rank).map(Vec::from)); + }, + Criterion::Desc(_) => { + group.sort_by_key(|d| std::cmp::Reverse(d.asc_desc_rank)); + new_groups.extend(group.linear_group_by_key(|d| d.asc_desc_rank).map(Vec::from)); + }, + } + } + groups = std::mem::take(&mut new_groups); + } + + if autorize_typo && optional_words { + groups.into_iter().flatten().collect() + } else if optional_words { + groups.into_iter().flatten().filter(|d| d.typo_rank == 0).collect() + } else if autorize_typo { + groups.into_iter().flatten().filter(|d| d.word_rank == 0).collect() + } else { + groups.into_iter().flatten().filter(|d| d.word_rank == 0 && d.typo_rank == 0).collect() + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TestDocument { + pub id: String, + pub word_rank: u32, + pub typo_rank: u32, + pub proximity_rank: u32, + pub attribute_rank: u32, + pub exact_rank: u32, + pub asc_desc_rank: u32, + pub title: String, + pub description: String, + pub tag: String, +} diff --git a/milli/tests/search/query_criteria.rs b/milli/tests/search/query_criteria.rs new file mode 100644 index 000000000..11d9b9e49 --- /dev/null +++ b/milli/tests/search/query_criteria.rs @@ -0,0 +1,531 @@ +use milli::{Search, SearchResult, Criterion}; +use big_s::S; + +use crate::search::{self, EXTERNAL_DOCUMENTS_IDS}; + +#[test] +fn none() { + let criteria = vec![]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn words() { + let criteria = vec![Criterion::Words]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn attribute() { + let criteria = vec![Criterion::Attribute]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn exactness() { + let criteria = vec![Criterion::Exactness]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn proximity() { + let criteria = vec![Criterion::Proximity]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn typo() { + let criteria = vec![Criterion::Typo]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn asc() { + let criteria = vec![Criterion::Asc(S("asc_desc_rank"))]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn desc() { + let criteria = vec![Criterion::Desc(S("asc_desc_rank"))]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn none_0_typo() { + let criteria = vec![]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.authorize_typos(false); + search.optional_words(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn attribute_0_typo() { + let criteria = vec![Criterion::Attribute]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + search.authorize_typos(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn exactness_0_typo() { + let criteria = vec![Criterion::Exactness]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + search.authorize_typos(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn proximity_0_typo() { + let criteria = vec![Criterion::Proximity]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + search.authorize_typos(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn typo_0_typo() { + let criteria = vec![Criterion::Typo]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + search.authorize_typos(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn asc_0_typo() { + let criteria = vec![Criterion::Asc(S("asc_desc_rank"))]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + search.authorize_typos(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn desc_0_typo() { + let criteria = vec![Criterion::Desc(S("asc_desc_rank"))]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + search.authorize_typos(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn test_desc_on_unexisting_field_should_return_all_1() { + let criteria = vec![Criterion::Desc(S("unexisting_field"))]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + search.authorize_typos(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let criteria = vec![]; + let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn test_asc_on_unexisting_field_should_return_all_1() { + let criteria = vec![Criterion::Asc(S("unexisting_field"))]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(false); + search.authorize_typos(false); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let criteria = vec![]; + let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn test_desc_on_unexisting_field_should_return_all_2() { + let criteria = vec![Criterion::Desc(S("unexisting_field"))]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let criteria = vec![]; + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn test_asc_on_unexisting_field_should_return_all_2() { + let criteria = vec![Criterion::Asc(S("unexisting_field"))]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let criteria = vec![]; + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); +} + +#[test] +fn criteria_mixup() { + use Criterion::*; + let index = search::setup_search_index_with_criteria(&vec![Words, Attribute, Desc(S("asc_desc_rank")), Exactness, Proximity, Typo]); + + + let criteria_mix = { + let desc = || Desc(S("asc_desc_rank")); + // all possible criteria order + vec![ + vec![Words, Attribute, desc(), Exactness, Proximity, Typo], + vec![Words, Attribute, desc(), Exactness, Typo, Proximity], + vec![Words, Attribute, desc(), Proximity, Exactness, Typo], + vec![Words, Attribute, desc(), Proximity, Typo, Exactness], + vec![Words, Attribute, desc(), Typo, Exactness, Proximity], + vec![Words, Attribute, desc(), Typo, Proximity, Exactness], + vec![Words, Attribute, Exactness, desc(), Proximity, Typo], + vec![Words, Attribute, Exactness, desc(), Typo, Proximity], + vec![Words, Attribute, Exactness, Proximity, desc(), Typo], + vec![Words, Attribute, Exactness, Proximity, Typo, desc()], + vec![Words, Attribute, Exactness, Typo, desc(), Proximity], + vec![Words, Attribute, Exactness, Typo, Proximity, desc()], + vec![Words, Attribute, Proximity, desc(), Exactness, Typo], + vec![Words, Attribute, Proximity, desc(), Typo, Exactness], + vec![Words, Attribute, Proximity, Exactness, desc(), Typo], + vec![Words, Attribute, Proximity, Exactness, Typo, desc()], + vec![Words, Attribute, Proximity, Typo, desc(), Exactness], + vec![Words, Attribute, Proximity, Typo, Exactness, desc()], + vec![Words, Attribute, Typo, desc(), Exactness, Proximity], + vec![Words, Attribute, Typo, desc(), Proximity, Exactness], + vec![Words, Attribute, Typo, Exactness, desc(), Proximity], + vec![Words, Attribute, Typo, Exactness, Proximity, desc()], + vec![Words, Attribute, Typo, Proximity, desc(), Exactness], + vec![Words, Attribute, Typo, Proximity, Exactness, desc()], + vec![Words, desc(), Attribute, Exactness, Proximity, Typo], + vec![Words, desc(), Attribute, Exactness, Typo, Proximity], + vec![Words, desc(), Attribute, Proximity, Exactness, Typo], + vec![Words, desc(), Attribute, Proximity, Typo, Exactness], + vec![Words, desc(), Attribute, Typo, Exactness, Proximity], + vec![Words, desc(), Attribute, Typo, Proximity, Exactness], + vec![Words, desc(), Exactness, Attribute, Proximity, Typo], + vec![Words, desc(), Exactness, Attribute, Typo, Proximity], + vec![Words, desc(), Exactness, Proximity, Attribute, Typo], + vec![Words, desc(), Exactness, Proximity, Typo, Attribute], + vec![Words, desc(), Exactness, Typo, Attribute, Proximity], + vec![Words, desc(), Exactness, Typo, Proximity, Attribute], + vec![Words, desc(), Proximity, Attribute, Exactness, Typo], + vec![Words, desc(), Proximity, Attribute, Typo, Exactness], + vec![Words, desc(), Proximity, Exactness, Attribute, Typo], + vec![Words, desc(), Proximity, Exactness, Typo, Attribute], + vec![Words, desc(), Proximity, Typo, Attribute, Exactness], + vec![Words, desc(), Proximity, Typo, Exactness, Attribute], + vec![Words, desc(), Typo, Attribute, Exactness, Proximity], + vec![Words, desc(), Typo, Attribute, Proximity, Exactness], + vec![Words, desc(), Typo, Exactness, Attribute, Proximity], + vec![Words, desc(), Typo, Exactness, Proximity, Attribute], + vec![Words, desc(), Typo, Proximity, Attribute, Exactness], + vec![Words, desc(), Typo, Proximity, Exactness, Attribute], + vec![Words, Exactness, Attribute, desc(), Proximity, Typo], + vec![Words, Exactness, Attribute, desc(), Typo, Proximity], + vec![Words, Exactness, Attribute, Proximity, desc(), Typo], + vec![Words, Exactness, Attribute, Proximity, Typo, desc()], + vec![Words, Exactness, Attribute, Typo, desc(), Proximity], + vec![Words, Exactness, Attribute, Typo, Proximity, desc()], + vec![Words, Exactness, desc(), Attribute, Proximity, Typo], + vec![Words, Exactness, desc(), Attribute, Typo, Proximity], + vec![Words, Exactness, desc(), Proximity, Attribute, Typo], + vec![Words, Exactness, desc(), Proximity, Typo, Attribute], + vec![Words, Exactness, desc(), Typo, Attribute, Proximity], + vec![Words, Exactness, desc(), Typo, Proximity, Attribute], + vec![Words, Exactness, Proximity, Attribute, desc(), Typo], + vec![Words, Exactness, Proximity, Attribute, Typo, desc()], + vec![Words, Exactness, Proximity, desc(), Attribute, Typo], + vec![Words, Exactness, Proximity, desc(), Typo, Attribute], + vec![Words, Exactness, Proximity, Typo, Attribute, desc()], + vec![Words, Exactness, Proximity, Typo, desc(), Attribute], + vec![Words, Exactness, Typo, Attribute, desc(), Proximity], + vec![Words, Exactness, Typo, Attribute, Proximity, desc()], + vec![Words, Exactness, Typo, desc(), Attribute, Proximity], + vec![Words, Exactness, Typo, desc(), Proximity, Attribute], + vec![Words, Exactness, Typo, Proximity, Attribute, desc()], + vec![Words, Exactness, Typo, Proximity, desc(), Attribute], + vec![Words, Proximity, Attribute, desc(), Exactness, Typo], + vec![Words, Proximity, Attribute, desc(), Typo, Exactness], + vec![Words, Proximity, Attribute, Exactness, desc(), Typo], + vec![Words, Proximity, Attribute, Exactness, Typo, desc()], + vec![Words, Proximity, Attribute, Typo, desc(), Exactness], + vec![Words, Proximity, Attribute, Typo, Exactness, desc()], + vec![Words, Proximity, desc(), Attribute, Exactness, Typo], + vec![Words, Proximity, desc(), Attribute, Typo, Exactness], + vec![Words, Proximity, desc(), Exactness, Attribute, Typo], + vec![Words, Proximity, desc(), Exactness, Typo, Attribute], + vec![Words, Proximity, desc(), Typo, Attribute, Exactness], + vec![Words, Proximity, desc(), Typo, Exactness, Attribute], + vec![Words, Proximity, Exactness, Attribute, desc(), Typo], + vec![Words, Proximity, Exactness, Attribute, Typo, desc()], + vec![Words, Proximity, Exactness, desc(), Attribute, Typo], + vec![Words, Proximity, Exactness, desc(), Typo, Attribute], + vec![Words, Proximity, Exactness, Typo, Attribute, desc()], + vec![Words, Proximity, Exactness, Typo, desc(), Attribute], + vec![Words, Proximity, Typo, Attribute, desc(), Exactness], + vec![Words, Proximity, Typo, Attribute, Exactness, desc()], + vec![Words, Proximity, Typo, desc(), Attribute, Exactness], + vec![Words, Proximity, Typo, desc(), Exactness, Attribute], + vec![Words, Proximity, Typo, Exactness, Attribute, desc()], + vec![Words, Proximity, Typo, Exactness, desc(), Attribute], + vec![Words, Typo, Attribute, desc(), Exactness, Proximity], + vec![Words, Typo, Attribute, desc(), Proximity, Exactness], + vec![Words, Typo, Attribute, Exactness, desc(), Proximity], + vec![Words, Typo, Attribute, Exactness, Proximity, desc()], + vec![Words, Typo, Attribute, Proximity, desc(), Exactness], + vec![Words, Typo, Attribute, Proximity, Exactness, desc()], + vec![Words, Typo, desc(), Attribute, Proximity, Exactness], + vec![Words, Typo, desc(), Exactness, Attribute, Proximity], + vec![Words, Typo, desc(), Exactness, Attribute, Proximity], + vec![Words, Typo, desc(), Exactness, Proximity, Attribute], + vec![Words, Typo, desc(), Proximity, Attribute, Exactness], + vec![Words, Typo, desc(), Proximity, Exactness, Attribute], + vec![Words, Typo, Exactness, Attribute, desc(), Proximity], + vec![Words, Typo, Exactness, Attribute, Proximity, desc()], + vec![Words, Typo, Exactness, desc(), Attribute, Proximity], + vec![Words, Typo, Exactness, desc(), Proximity, Attribute], + vec![Words, Typo, Exactness, Proximity, Attribute, desc()], + vec![Words, Typo, Exactness, Proximity, desc(), Attribute], + vec![Words, Typo, Proximity, Attribute, desc(), Exactness], + vec![Words, Typo, Proximity, Attribute, Exactness, desc()], + vec![Words, Typo, Proximity, desc(), Attribute, Exactness], + vec![Words, Typo, Proximity, desc(), Exactness, Attribute], + vec![Words, Typo, Proximity, Exactness, Attribute, desc()], + vec![Words, Typo, Proximity, Exactness, desc(), Attribute], + ] + }; + + for criteria in criteria_mix { + eprintln!("Testing with criteria order: {:?}", &criteria); + //update criteria + let mut wtxn = index.write_txn().unwrap(); + index.put_criteria(&mut wtxn, &criteria).unwrap(); + wtxn.commit().unwrap(); + + let mut rtxn = index.read_txn().unwrap(); + + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.optional_words(true); + search.authorize_typos(true); + + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + + let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); + } +} From 1fcc5f73acac427234d5c0d425add2cbb5613da3 Mon Sep 17 00:00:00 2001 From: many Date: Tue, 8 Jun 2021 12:33:02 +0200 Subject: [PATCH 2/8] Factorize tests using macro_rules --- milli/tests/search/mod.rs | 5 +- milli/tests/search/query_criteria.rs | 403 +++------------------------ 2 files changed, 45 insertions(+), 363 deletions(-) diff --git a/milli/tests/search/mod.rs b/milli/tests/search/mod.rs index 609787712..60ad3d45f 100644 --- a/milli/tests/search/mod.rs +++ b/milli/tests/search/mod.rs @@ -98,14 +98,15 @@ pub fn expected_order(criteria: &[Criterion], autorize_typo: bool, optional_word group.sort_by_key(|d| d.word_rank); new_groups.extend(group.linear_group_by_key(|d| d.word_rank).map(Vec::from)); }, - Criterion::Asc(_) => { + Criterion::Asc(field_name) if field_name == "asc_desc_rank" => { group.sort_by_key(|d| d.asc_desc_rank); new_groups.extend(group.linear_group_by_key(|d| d.asc_desc_rank).map(Vec::from)); }, - Criterion::Desc(_) => { + Criterion::Desc(field_name) if field_name == "asc_desc_rank" => { group.sort_by_key(|d| std::cmp::Reverse(d.asc_desc_rank)); new_groups.extend(group.linear_group_by_key(|d| d.asc_desc_rank).map(Vec::from)); }, + Criterion::Asc(_) | Criterion::Desc(_) => new_groups.push(group.clone()), } } groups = std::mem::take(&mut new_groups); diff --git a/milli/tests/search/query_criteria.rs b/milli/tests/search/query_criteria.rs index 11d9b9e49..7e398fceb 100644 --- a/milli/tests/search/query_criteria.rs +++ b/milli/tests/search/query_criteria.rs @@ -2,376 +2,57 @@ use milli::{Search, SearchResult, Criterion}; use big_s::S; use crate::search::{self, EXTERNAL_DOCUMENTS_IDS}; +use Criterion::*; -#[test] -fn none() { - let criteria = vec![]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); +const ALLOW_TYPOS: bool = true; +const DISALLOW_TYPOS: bool = false; +const ALLOW_OPTIONAL_WORDS: bool = true; +const DISALLOW_OPTIONAL_WORDS: bool = false; - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); +macro_rules! test_criterion { + ($func:ident, $optional_word:ident, $authorize_typos:ident $(, $criterion:expr)?) => { + #[test] + fn $func() { + let criteria = vec![$($criterion)?]; + let index = search::setup_search_index_with_criteria(&criteria); + let mut rtxn = index.read_txn().unwrap(); - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + let mut search = Search::new(&mut rtxn, &index); + search.query(search::TEST_QUERY); + search.limit(EXTERNAL_DOCUMENTS_IDS.len()); + search.authorize_typos($authorize_typos); + search.optional_words($optional_word); - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - assert_eq!(documents_ids, expected_external_ids); + let expected_external_ids: Vec<_> = search::expected_order(&criteria, $authorize_typos, $optional_word).into_iter().map(|d| d.id).collect(); + let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + assert_eq!(documents_ids, expected_external_ids); + + } + } } -#[test] -fn words() { - let criteria = vec![Criterion::Words]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn attribute() { - let criteria = vec![Criterion::Attribute]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn exactness() { - let criteria = vec![Criterion::Exactness]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn proximity() { - let criteria = vec![Criterion::Proximity]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn typo() { - let criteria = vec![Criterion::Typo]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn asc() { - let criteria = vec![Criterion::Asc(S("asc_desc_rank"))]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn desc() { - let criteria = vec![Criterion::Desc(S("asc_desc_rank"))]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn none_0_typo() { - let criteria = vec![]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.authorize_typos(false); - search.optional_words(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn attribute_0_typo() { - let criteria = vec![Criterion::Attribute]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - search.authorize_typos(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn exactness_0_typo() { - let criteria = vec![Criterion::Exactness]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - search.authorize_typos(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn proximity_0_typo() { - let criteria = vec![Criterion::Proximity]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - search.authorize_typos(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn typo_0_typo() { - let criteria = vec![Criterion::Typo]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - search.authorize_typos(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn asc_0_typo() { - let criteria = vec![Criterion::Asc(S("asc_desc_rank"))]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - search.authorize_typos(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn desc_0_typo() { - let criteria = vec![Criterion::Desc(S("asc_desc_rank"))]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - search.authorize_typos(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn test_desc_on_unexisting_field_should_return_all_1() { - let criteria = vec![Criterion::Desc(S("unexisting_field"))]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - search.authorize_typos(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let criteria = vec![]; - let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn test_asc_on_unexisting_field_should_return_all_1() { - let criteria = vec![Criterion::Asc(S("unexisting_field"))]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(false); - search.authorize_typos(false); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let criteria = vec![]; - let expected_external_ids: Vec<_> = search::expected_order(&criteria, false, false).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn test_desc_on_unexisting_field_should_return_all_2() { - let criteria = vec![Criterion::Desc(S("unexisting_field"))]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let criteria = vec![]; - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} - -#[test] -fn test_asc_on_unexisting_field_should_return_all_2() { - let criteria = vec![Criterion::Asc(S("unexisting_field"))]; - let index = search::setup_search_index_with_criteria(&criteria); - let mut rtxn = index.read_txn().unwrap(); - - let mut search = Search::new(&mut rtxn, &index); - search.query(search::TEST_QUERY); - search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); - - let criteria = vec![]; - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - - assert_eq!(documents_ids, expected_external_ids); -} +test_criterion!(none_allow_typo, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS); +test_criterion!(none_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS); +test_criterion!(words_allow_typo, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Words); +test_criterion!(attribute_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Attribute); +test_criterion!(attribute_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Attribute); +test_criterion!(exactness_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Exactness); +test_criterion!(exactness_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Exactness); +test_criterion!(proximity_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Proximity); +test_criterion!(proximity_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Proximity); +test_criterion!(asc_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Asc(S("asc_desc_rank"))); +test_criterion!(asc_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Asc(S("asc_desc_rank"))); +test_criterion!(desc_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Desc(S("asc_desc_rank"))); +test_criterion!(desc_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Desc(S("asc_desc_rank"))); +test_criterion!(asc_unexisting_field_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Asc(S("unexisting_field"))); +test_criterion!(asc_unexisting_field_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Asc(S("unexisting_field"))); +test_criterion!(desc_unexisting_field_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Desc(S("unexisting_field"))); +test_criterion!(desc_unexisting_field_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Desc(S("unexisting_field"))); #[test] fn criteria_mixup() { From b64cd2a3e361d43c67d612d045b22b6feeba56b5 Mon Sep 17 00:00:00 2001 From: many Date: Tue, 8 Jun 2021 14:11:00 +0200 Subject: [PATCH 3/8] Resolve PR comments --- milli/tests/search/mod.rs | 13 ++----- milli/tests/search/query_criteria.rs | 53 ++++++++++++++-------------- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/milli/tests/search/mod.rs b/milli/tests/search/mod.rs index 60ad3d45f..98102f9e9 100644 --- a/milli/tests/search/mod.rs +++ b/milli/tests/search/mod.rs @@ -52,13 +52,6 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { index } -#[allow(dead_code)] -pub fn external_to_internal_ids(index: &Index, external_ids: &[&str]) -> Vec { - let mut rtxn = index.read_txn().unwrap(); - let docid_map = index.external_documents_ids(&mut rtxn).unwrap(); - external_ids.iter().map(|id| docid_map.get(id).unwrap()).collect() -} - pub fn internal_to_external_ids(index: &Index, internal_ids: &[DocumentId]) -> Vec { let mut rtxn = index.read_txn().unwrap(); let docid_map = index.external_documents_ids(&mut rtxn).unwrap(); @@ -70,7 +63,7 @@ fn fetch_dataset() -> Vec { serde_json::Deserializer::from_str(CONTENT).into_iter().map(|r| r.unwrap()).collect() } -pub fn expected_order(criteria: &[Criterion], autorize_typo: bool, optional_words: bool) -> Vec { +pub fn expected_order(criteria: &[Criterion], authorize_typo: bool, optional_words: bool) -> Vec { let dataset = fetch_dataset(); let mut groups: Vec> = vec![dataset]; @@ -112,11 +105,11 @@ pub fn expected_order(criteria: &[Criterion], autorize_typo: bool, optional_word groups = std::mem::take(&mut new_groups); } - if autorize_typo && optional_words { + if authorize_typo && optional_words { groups.into_iter().flatten().collect() } else if optional_words { groups.into_iter().flatten().filter(|d| d.typo_rank == 0).collect() - } else if autorize_typo { + } else if authorize_typo { groups.into_iter().flatten().filter(|d| d.word_rank == 0).collect() } else { groups.into_iter().flatten().filter(|d| d.word_rank == 0 && d.typo_rank == 0).collect() diff --git a/milli/tests/search/query_criteria.rs b/milli/tests/search/query_criteria.rs index 7e398fceb..160b77c5b 100644 --- a/milli/tests/search/query_criteria.rs +++ b/milli/tests/search/query_criteria.rs @@ -23,36 +23,34 @@ macro_rules! test_criterion { search.authorize_typos($authorize_typos); search.optional_words($optional_word); - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + let SearchResult { documents_ids, .. } = search.execute().unwrap(); - let expected_external_ids: Vec<_> = search::expected_order(&criteria, $authorize_typos, $optional_word).into_iter().map(|d| d.id).collect(); + let expected_external_ids: Vec<_> = search::expected_order(&criteria, $authorize_typos, $optional_word) + .into_iter() + .map(|d| d.id).collect(); let documents_ids = search::internal_to_external_ids(&index, &documents_ids); - assert_eq!(documents_ids, expected_external_ids); - } } } - - -test_criterion!(none_allow_typo, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS); -test_criterion!(none_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS); -test_criterion!(words_allow_typo, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Words); -test_criterion!(attribute_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Attribute); -test_criterion!(attribute_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Attribute); -test_criterion!(exactness_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Exactness); -test_criterion!(exactness_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Exactness); -test_criterion!(proximity_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Proximity); -test_criterion!(proximity_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Proximity); -test_criterion!(asc_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Asc(S("asc_desc_rank"))); -test_criterion!(asc_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Asc(S("asc_desc_rank"))); -test_criterion!(desc_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Desc(S("asc_desc_rank"))); -test_criterion!(desc_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Desc(S("asc_desc_rank"))); -test_criterion!(asc_unexisting_field_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Asc(S("unexisting_field"))); -test_criterion!(asc_unexisting_field_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Asc(S("unexisting_field"))); -test_criterion!(desc_unexisting_field_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Desc(S("unexisting_field"))); -test_criterion!(desc_unexisting_field_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Desc(S("unexisting_field"))); +test_criterion!(none_allow_typo, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS); +test_criterion!(none_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS); +test_criterion!(words_allow_typo, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Words); +test_criterion!(attribute_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Attribute); +test_criterion!(attribute_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Attribute); +test_criterion!(exactness_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Exactness); +test_criterion!(exactness_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Exactness); +test_criterion!(proximity_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Proximity); +test_criterion!(proximity_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Proximity); +test_criterion!(asc_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Asc(S("asc_desc_rank"))); +test_criterion!(asc_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Asc(S("asc_desc_rank"))); +test_criterion!(desc_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Desc(S("asc_desc_rank"))); +test_criterion!(desc_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Desc(S("asc_desc_rank"))); +test_criterion!(asc_unexisting_field_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Asc(S("unexisting_field"))); +test_criterion!(asc_unexisting_field_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Asc(S("unexisting_field"))); +test_criterion!(desc_unexisting_field_allow_typo, DISALLOW_OPTIONAL_WORDS, ALLOW_TYPOS, Desc(S("unexisting_field"))); +test_criterion!(desc_unexisting_field_disallow_typo, DISALLOW_OPTIONAL_WORDS, DISALLOW_TYPOS, Desc(S("unexisting_field"))); #[test] fn criteria_mixup() { @@ -61,6 +59,7 @@ fn criteria_mixup() { let criteria_mix = { + // Criterion doesn't implement Copy, we create a new Criterion using a closure let desc = || Desc(S("asc_desc_rank")); // all possible criteria order vec![ @@ -199,12 +198,12 @@ fn criteria_mixup() { let mut search = Search::new(&mut rtxn, &index); search.query(search::TEST_QUERY); search.limit(EXTERNAL_DOCUMENTS_IDS.len()); - search.optional_words(true); - search.authorize_typos(true); + search.optional_words(ALLOW_OPTIONAL_WORDS); + search.authorize_typos(ALLOW_TYPOS); - let SearchResult { matching_words: _matching_words, candidates: _candidates, documents_ids } = search.execute().unwrap(); + let SearchResult { documents_ids, .. } = search.execute().unwrap(); - let expected_external_ids: Vec<_> = search::expected_order(&criteria, true, true).into_iter().map(|d| d.id).collect(); + let expected_external_ids: Vec<_> = search::expected_order(&criteria, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS).into_iter().map(|d| d.id).collect(); let documents_ids = search::internal_to_external_ids(&index, &documents_ids); assert_eq!(documents_ids, expected_external_ids); From afb09c914df05a3579d139b919a98bfa4c851d29 Mon Sep 17 00:00:00 2001 From: Many Date: Tue, 8 Jun 2021 14:24:17 +0200 Subject: [PATCH 4/8] Update milli/tests/search/query_criteria.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Renault --- milli/tests/search/mod.rs | 2 -- milli/tests/search/query_criteria.rs | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/milli/tests/search/mod.rs b/milli/tests/search/mod.rs index 98102f9e9..20cf13034 100644 --- a/milli/tests/search/mod.rs +++ b/milli/tests/search/mod.rs @@ -29,9 +29,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { builder.set_criteria(criteria); builder.set_filterable_fields(hashset!{ S("tag"), - S("unexisting_field"), S("asc_desc_rank"), - S("unexisting_field"), }); builder.set_synonyms(hashmap!{ S("hello") => vec![S("good morning")], diff --git a/milli/tests/search/query_criteria.rs b/milli/tests/search/query_criteria.rs index 160b77c5b..b6f692674 100644 --- a/milli/tests/search/query_criteria.rs +++ b/milli/tests/search/query_criteria.rs @@ -203,7 +203,10 @@ fn criteria_mixup() { let SearchResult { documents_ids, .. } = search.execute().unwrap(); - let expected_external_ids: Vec<_> = search::expected_order(&criteria, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS).into_iter().map(|d| d.id).collect(); + let expected_external_ids: Vec<_> = search::expected_order(&criteria, ALLOW_OPTIONAL_WORDS, ALLOW_TYPOS) + .into_iter() + .map(|d| d.id) + .collect(); let documents_ids = search::internal_to_external_ids(&index, &documents_ids); assert_eq!(documents_ids, expected_external_ids); From 103dddba2fab450bd38ca398a37696564d065cd2 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 8 Jun 2021 17:44:25 +0200 Subject: [PATCH 5/8] Move the UpdateStore into the http-ui crate --- Cargo.lock | 2 +- http-ui/Cargo.toml | 1 + http-ui/src/main.rs | 6 +++++- {milli => http-ui}/src/update_store.rs | 4 +++- milli/Cargo.toml | 1 - milli/src/lib.rs | 2 -- 6 files changed, 10 insertions(+), 6 deletions(-) rename {milli => http-ui}/src/update_store.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 575f582bc..e3c00257d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,6 +961,7 @@ dependencies = [ "askama_warp", "byte-unit", "bytes 0.5.6", + "crossbeam-channel", "either", "flate2", "fst", @@ -1365,7 +1366,6 @@ dependencies = [ "bstr", "byteorder", "chrono", - "crossbeam-channel", "csv", "either", "flate2", diff --git a/http-ui/Cargo.toml b/http-ui/Cargo.toml index de95ce3a6..970d1d9bf 100644 --- a/http-ui/Cargo.toml +++ b/http-ui/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" [dependencies] anyhow = "1.0.38" byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } +crossbeam-channel = "0.5.0" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } heed = "0.10.6" meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } diff --git a/http-ui/src/main.rs b/http-ui/src/main.rs index b6a894373..1f91e6370 100644 --- a/http-ui/src/main.rs +++ b/http-ui/src/main.rs @@ -1,3 +1,5 @@ +mod update_store; + use std::{io, mem}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt::Display; @@ -29,10 +31,12 @@ use tokio::sync::broadcast; use warp::{Filter, http::Response}; use warp::filters::ws::Message; -use milli::{FilterCondition, Index, MatchingWords, obkv_to_json, SearchResult, UpdateStore}; +use milli::{FilterCondition, Index, MatchingWords, obkv_to_json, SearchResult}; use milli::update::{IndexDocumentsMethod, Setting, UpdateBuilder, UpdateFormat}; use milli::update::UpdateIndexingStep::*; +use self::update_store::UpdateStore; + static GLOBAL_THREAD_POOL: OnceCell = OnceCell::new(); #[derive(Debug, StructOpt)] diff --git a/milli/src/update_store.rs b/http-ui/src/update_store.rs similarity index 99% rename from milli/src/update_store.rs rename to http-ui/src/update_store.rs index 7211a6293..122ee6031 100644 --- a/milli/src/update_store.rs +++ b/http-ui/src/update_store.rs @@ -1,3 +1,5 @@ +#![allow(unused)] + use std::path::Path; use std::sync::Arc; @@ -6,7 +8,7 @@ use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; use heed::{EnvOpenOptions, Env, Database}; use serde::{Serialize, Deserialize}; -use crate::BEU64; +pub type BEU64 = heed::zerocopy::U64; #[derive(Clone)] pub struct UpdateStore { diff --git a/milli/Cargo.toml b/milli/Cargo.toml index abcded8c9..3e755d088 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -9,7 +9,6 @@ anyhow = "1.0.38" bstr = "0.2.15" byteorder = "1.4.2" chrono = { version = "0.4.19", features = ["serde"] } -crossbeam-channel = "0.5.0" csv = "1.1.5" either = "1.6.1" flate2 = "1.0.20" diff --git a/milli/src/lib.rs b/milli/src/lib.rs index 39e107073..8c1ed514c 100644 --- a/milli/src/lib.rs +++ b/milli/src/lib.rs @@ -4,7 +4,6 @@ mod criterion; mod external_documents_ids; mod fields_ids_map; mod search; -mod update_store; pub mod facet; pub mod heed_codec; pub mod index; @@ -29,7 +28,6 @@ pub use self::heed_codec::{RoaringBitmapLenCodec, BoRoaringBitmapLenCodec, CboRo pub use self::index::Index; pub use self::search::{Search, FacetDistribution, FilterCondition, SearchResult, MatchingWords}; pub use self::tree_level::TreeLevel; -pub use self::update_store::UpdateStore; pub type FastMap4 = HashMap>; pub type FastMap8 = HashMap>; From 7e93811fbcb311cce5085f92bf0e6a507d182577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= Date: Tue, 8 Jun 2021 18:18:54 +0200 Subject: [PATCH 6/8] Update dataset links --- benchmarks/README.md | 8 ++++---- benchmarks/build.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index e3df18db3..843ea9b29 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -87,7 +87,7 @@ The benchmarks are available for the following datasets: ### Songs -`songs` is a subset of the [`songs.csv` dataset](https://meili-datasets.s3.fr-par.scw.cloud/songs.csv.gz). +`songs` is a subset of the [`songs.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/songs.csv.gz). It was generated with this command: @@ -95,11 +95,11 @@ It was generated with this command: xsv sample --seed 42 1000000 songs.csv -o smol-songs.csv ``` -_[Download the generated `songs` dataset](https://meili-datasets.s3.fr-par.scw.cloud/benchmarks/smol-songs.csv.gz)._ +_[Download the generated `songs` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-songs.csv.gz)._ ### Wiki -`wiki` is a subset of the [`wikipedia-articles.csv` dataset](https://meili-datasets.s3.fr-par.scw.cloud/wiki-articles.csv.gz). +`wiki` is a subset of the [`wikipedia-articles.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/wiki-articles.csv.gz). It was generated with the following command: @@ -107,5 +107,5 @@ It was generated with the following command: xsv sample --seed 42 500000 wiki-articles.csv -o smol-wiki-articles.csv ``` -_[Download the generated `wiki` dataset](https://meili-datasets.s3.fr-par.scw.cloud/benchmarks/smol-wiki-articles.csv.gz)._ +_[Download the generated `wiki` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-wiki-articles.csv.gz)._ diff --git a/benchmarks/build.rs b/benchmarks/build.rs index dc92a1a4c..58300bab9 100644 --- a/benchmarks/build.rs +++ b/benchmarks/build.rs @@ -10,7 +10,7 @@ use convert_case::{Case, Casing}; use flate2::read::GzDecoder; use reqwest::IntoUrl; -const BASE_URL: &str = "https://meili-datasets.s3.fr-par.scw.cloud/benchmarks"; +const BASE_URL: &str = "https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets"; const DATASET_SONGS: &str = "smol-songs"; const DATASET_WIKI: &str = "smol-wiki-articles"; From ab696f6a2326e069b372d507dab52678409252a6 Mon Sep 17 00:00:00 2001 From: Many Date: Wed, 9 Jun 2021 10:12:17 +0200 Subject: [PATCH 7/8] Update milli/tests/search/query_criteria.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Renault --- milli/tests/search/query_criteria.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/milli/tests/search/query_criteria.rs b/milli/tests/search/query_criteria.rs index b6f692674..f0eecfaba 100644 --- a/milli/tests/search/query_criteria.rs +++ b/milli/tests/search/query_criteria.rs @@ -57,7 +57,6 @@ fn criteria_mixup() { use Criterion::*; let index = search::setup_search_index_with_criteria(&vec![Words, Attribute, Desc(S("asc_desc_rank")), Exactness, Proximity, Typo]); - let criteria_mix = { // Criterion doesn't implement Copy, we create a new Criterion using a closure let desc = || Desc(S("asc_desc_rank")); From f4ff30e99d09438715605085fade28ef1822b807 Mon Sep 17 00:00:00 2001 From: Many Date: Wed, 9 Jun 2021 10:12:24 +0200 Subject: [PATCH 8/8] Update milli/tests/search/mod.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Renault --- milli/tests/search/mod.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/milli/tests/search/mod.rs b/milli/tests/search/mod.rs index 20cf13034..8c63e5e08 100644 --- a/milli/tests/search/mod.rs +++ b/milli/tests/search/mod.rs @@ -57,12 +57,8 @@ pub fn internal_to_external_ids(index: &Index, internal_ids: &[DocumentId]) -> V internal_ids.iter().map(|id| docid_map.get(id).unwrap().to_string()).collect() } -fn fetch_dataset() -> Vec { - serde_json::Deserializer::from_str(CONTENT).into_iter().map(|r| r.unwrap()).collect() -} - pub fn expected_order(criteria: &[Criterion], authorize_typo: bool, optional_words: bool) -> Vec { - let dataset = fetch_dataset(); + let dataset = serde_json::Deserializer::from_str(CONTENT).into_iter().map(|r| r.unwrap()).collect(); let mut groups: Vec> = vec![dataset]; for criterion in criteria {