snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 18:45:06 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Fix compile error+warnings in new tests

Browse Source
This commit is contained in:
Loïc Lecrenier 2022-08-31 14:19:52 +02:00 committed by Loïc Lecrenier
parent e8a156d682
commit d30c89e345
8 changed files with 67 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
milli/src/search/facet/facet_distribution_iter.rs
View File

@ -109,7 +109,7 @@ where
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use heed::BytesDecode; use heed::BytesDecode;
use rand::{rngs::SmallRng, Rng, SeedableRng}; use rand::{Rng, SeedableRng};
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use std::ops::ControlFlow; use std::ops::ControlFlow;
@ -125,7 +125,7 @@ mod tests {
for i in 0..256u16 { for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new(); let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32); bitmap.insert(i as u32);
index.insert(&mut txn, 0, &i, &bitmap); index.insert(&mut txn, 0, &(i as f64), &bitmap);
} }
txn.commit().unwrap(); txn.commit().unwrap();
index index
@ -134,14 +134,14 @@ mod tests {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8); let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut txn = index.env.write_txn().unwrap(); let mut txn = index.env.write_txn().unwrap();
let rng = rand::rngs::SmallRng::from_seed([0; 32]); let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
let keys = let keys =
std::iter::from_fn(|| Some(rng.gen_range(0..256))).take(128).collect::<Vec<u32>>(); std::iter::from_fn(|| Some(rng.gen_range(0..256))).take(128).collect::<Vec<u32>>();
for (_i, key) in keys.into_iter().enumerate() { for (_i, key) in keys.into_iter().enumerate() {
let mut bitmap = RoaringBitmap::new(); let mut bitmap = RoaringBitmap::new();
bitmap.insert(key); bitmap.insert(key);
bitmap.insert(key + 100.); bitmap.insert(key + 100);
index.insert(&mut txn, 0, &(key as f64), &bitmap); index.insert(&mut txn, 0, &(key as f64), &bitmap);
} }
txn.commit().unwrap(); txn.commit().unwrap();
@ -156,13 +156,13 @@ mod tests {
#[test] #[test]
fn filter_distribution_all() { fn filter_distribution_all() {
let indexes = [get_simple_index(), get_random_looking_index()]; let indexes = [get_simple_index(), get_random_looking_index()];
for (i, index) in indexes.into_iter().enumerate() { for (i, index) in indexes.iter().enumerate() {
let txn = index.env.read_txn().unwrap(); let txn = index.env.read_txn().unwrap();
let candidates = (0..=255).into_iter().collect::<RoaringBitmap>(); let candidates = (0..=255).into_iter().collect::<RoaringBitmap>();
let mut results = String::new(); let mut results = String::new();
iterate_over_facet_distribution( iterate_over_facet_distribution(
&txn, &txn,
&index.db.content, index.db.content,
0, 0,
&candidates, &candidates,
|facet, count| { |facet, count| {
@ -170,7 +170,8 @@ mod tests {
results.push_str(&format!("{facet}: {count}\n")); results.push_str(&format!("{facet}: {count}\n"));
ControlFlow::Continue(()) ControlFlow::Continue(())
}, },
); )
.unwrap();
insta::assert_snapshot!(format!("filter_distribution_{i}_all"), results); insta::assert_snapshot!(format!("filter_distribution_{i}_all"), results);
txn.commit().unwrap(); txn.commit().unwrap();
@ -179,14 +180,14 @@ mod tests {
#[test] #[test]
fn filter_distribution_all_stop_early() { fn filter_distribution_all_stop_early() {
let indexes = [get_simple_index(), get_random_looking_index()]; let indexes = [get_simple_index(), get_random_looking_index()];
for (i, index) in indexes.into_iter().enumerate() { for (i, index) in indexes.iter().enumerate() {
let txn = index.env.read_txn().unwrap(); let txn = index.env.read_txn().unwrap();
let candidates = (0..=255).into_iter().collect::<RoaringBitmap>(); let candidates = (0..=255).into_iter().collect::<RoaringBitmap>();
let mut results = String::new(); let mut results = String::new();
let mut nbr_facets = 0; let mut nbr_facets = 0;
iterate_over_facet_distribution( iterate_over_facet_distribution(
&txn, &txn,
&index.db.content, index.db.content,
0, 0,
&candidates, &candidates,
|facet, count| { |facet, count| {
@ -200,7 +201,8 @@ mod tests {
ControlFlow::Continue(()) ControlFlow::Continue(())
} }
}, },
); )
.unwrap();
insta::assert_snapshot!(format!("filter_distribution_{i}_all_stop_early"), results); insta::assert_snapshot!(format!("filter_distribution_{i}_all_stop_early"), results);
txn.commit().unwrap(); txn.commit().unwrap();

25
milli/src/search/facet/facet_range_search.rs
View File

@ -259,8 +259,9 @@ impl<'t, 'b, 'bitmap> FacetRangeSearch<'t, 'b, 'bitmap> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::{ use crate::{
heed_codec::facet::new::ordered_f64_codec::OrderedF64Codec, heed_codec::facet::new::{ordered_f64_codec::OrderedF64Codec, FacetKeyCodec},
search::facet::test::FacetIndex, snapshot_tests::display_bitmap, search::facet::test::FacetIndex,
snapshot_tests::display_bitmap,
}; };
use rand::{Rng, SeedableRng}; use rand::{Rng, SeedableRng};
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
@ -283,7 +284,7 @@ mod tests {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8); let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut txn = index.env.write_txn().unwrap(); let mut txn = index.env.write_txn().unwrap();
let rng = rand::rngs::SmallRng::from_seed([0; 32]); let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
let keys = let keys =
std::iter::from_fn(|| Some(rng.gen_range(0..256))).take(128).collect::<Vec<u32>>(); std::iter::from_fn(|| Some(rng.gen_range(0..256))).take(128).collect::<Vec<u32>>();
@ -305,7 +306,7 @@ mod tests {
#[test] #[test]
fn filter_range_increasing() { fn filter_range_increasing() {
let indexes = [get_simple_index(), get_random_looking_index()]; let indexes = [get_simple_index(), get_random_looking_index()];
for (i, index) in indexes.into_iter().enumerate() { for (i, index) in indexes.iter().enumerate() {
let txn = index.env.read_txn().unwrap(); let txn = index.env.read_txn().unwrap();
let mut results = String::new(); let mut results = String::new();
for i in 0..=255 { for i in 0..=255 {
@ -314,7 +315,7 @@ mod tests {
let end = Bound::Included(i); let end = Bound::Included(i);
let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>( let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>(
&txn, &txn,
&index.db.content, index.db.content.remap_key_type::<FacetKeyCodec<OrderedF64Codec>>(),
0, 0,
&start, &start,
&end, &end,
@ -333,7 +334,7 @@ mod tests {
let end = Bound::Excluded(i); let end = Bound::Excluded(i);
let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>( let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>(
&txn, &txn,
&index.db.content, index.db.content.remap_key_type::<FacetKeyCodec<OrderedF64Codec>>(),
0, 0,
&start, &start,
&end, &end,
@ -351,7 +352,7 @@ mod tests {
#[test] #[test]
fn filter_range_decreasing() { fn filter_range_decreasing() {
let indexes = [get_simple_index(), get_random_looking_index()]; let indexes = [get_simple_index(), get_random_looking_index()];
for (i, index) in indexes.into_iter().enumerate() { for (i, index) in indexes.iter().enumerate() {
let txn = index.env.read_txn().unwrap(); let txn = index.env.read_txn().unwrap();
let mut results = String::new(); let mut results = String::new();
@ -362,7 +363,7 @@ mod tests {
let end = Bound::Included(255.); let end = Bound::Included(255.);
let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>( let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>(
&txn, &txn,
&index.db.content, index.db.content.remap_key_type::<FacetKeyCodec<OrderedF64Codec>>(),
0, 0,
&start, &start,
&end, &end,
@ -384,7 +385,7 @@ mod tests {
let end = Bound::Excluded(255.); let end = Bound::Excluded(255.);
let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>( let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>(
&txn, &txn,
&index.db.content, index.db.content.remap_key_type::<FacetKeyCodec<OrderedF64Codec>>(),
0, 0,
&start, &start,
&end, &end,
@ -404,7 +405,7 @@ mod tests {
#[test] #[test]
fn filter_range_pinch() { fn filter_range_pinch() {
let indexes = [get_simple_index(), get_random_looking_index()]; let indexes = [get_simple_index(), get_random_looking_index()];
for (i, index) in indexes.into_iter().enumerate() { for (i, index) in indexes.iter().enumerate() {
let txn = index.env.read_txn().unwrap(); let txn = index.env.read_txn().unwrap();
let mut results = String::new(); let mut results = String::new();
@ -415,7 +416,7 @@ mod tests {
let end = Bound::Included(255. - i); let end = Bound::Included(255. - i);
let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>( let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>(
&txn, &txn,
&index.db.content, index.db.content.remap_key_type::<FacetKeyCodec<OrderedF64Codec>>(),
0, 0,
&start, &start,
&end, &end,
@ -434,7 +435,7 @@ mod tests {
let end = Bound::Excluded(255. - i); let end = Bound::Excluded(255. - i);
let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>( let docids = find_docids_of_facet_within_bounds::<OrderedF64Codec>(
&txn, &txn,
&index.db.content, index.db.content.remap_key_type::<FacetKeyCodec<OrderedF64Codec>>(),
0, 0,
&start, &start,
&end, &end,

15
milli/src/search/facet/facet_sort_ascending.rs
View File

@ -83,7 +83,6 @@ impl<'t, 'e> Iterator for AscendingFacetSort<'t, 'e> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use heed::BytesDecode;
use rand::Rng; use rand::Rng;
use rand::SeedableRng; use rand::SeedableRng;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
@ -100,7 +99,7 @@ mod tests {
for i in 0..256u16 { for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new(); let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32); bitmap.insert(i as u32);
index.insert(&mut txn, 0, &i, &bitmap); index.insert(&mut txn, 0, &(i as f64), &bitmap);
} }
txn.commit().unwrap(); txn.commit().unwrap();
index index
@ -109,7 +108,7 @@ mod tests {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8); let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut txn = index.env.write_txn().unwrap(); let mut txn = index.env.write_txn().unwrap();
let rng = rand::rngs::SmallRng::from_seed([0; 32]); let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
let keys = let keys =
std::iter::from_fn(|| Some(rng.gen_range(0..256))).take(128).collect::<Vec<u32>>(); std::iter::from_fn(|| Some(rng.gen_range(0..256))).take(128).collect::<Vec<u32>>();
@ -131,14 +130,14 @@ mod tests {
#[test] #[test]
fn filter_sort() { fn filter_sort() {
let indexes = [get_simple_index(), get_random_looking_index()]; let indexes = [get_simple_index(), get_random_looking_index()];
for (i, index) in indexes.into_iter().enumerate() { for (i, index) in indexes.iter().enumerate() {
let txn = index.env.read_txn().unwrap(); let txn = index.env.read_txn().unwrap();
let candidates = (200..=300).into_iter().collect::<RoaringBitmap>(); let candidates = (200..=300).into_iter().collect::<RoaringBitmap>();
let mut results = String::new(); let mut results = String::new();
let iter = ascending_facet_sort(&txn, &index.db.content, 0, candidates); let iter = ascending_facet_sort(&txn, index.db.content, 0, candidates).unwrap();
for (facet, docids) in iter { for el in iter {
let facet = OrderedF64Codec::bytes_decode(facet).unwrap(); let docids = el.unwrap();
results.push_str(&format!("{facet}: {}\n", display_bitmap(&docids))); results.push_str(&display_bitmap(&docids));
} }
insta::assert_snapshot!(format!("filter_sort_{i}_ascending"), results); insta::assert_snapshot!(format!("filter_sort_{i}_ascending"), results);

18
milli/src/search/facet/facet_sort_descending.rs
View File

@ -111,8 +111,6 @@ impl<'t> Iterator for DescendingFacetSort<'t> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use heed::BytesDecode;
use rand::Rng; use rand::Rng;
use rand::SeedableRng; use rand::SeedableRng;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
@ -129,7 +127,7 @@ mod tests {
for i in 0..256u16 { for i in 0..256u16 {
let mut bitmap = RoaringBitmap::new(); let mut bitmap = RoaringBitmap::new();
bitmap.insert(i as u32); bitmap.insert(i as u32);
index.insert(&mut txn, 0, &i, &bitmap); index.insert(&mut txn, 0, &(i as f64), &bitmap);
} }
txn.commit().unwrap(); txn.commit().unwrap();
index index
@ -138,14 +136,14 @@ mod tests {
let index = FacetIndex::<OrderedF64Codec>::new(4, 8); let index = FacetIndex::<OrderedF64Codec>::new(4, 8);
let mut txn = index.env.write_txn().unwrap(); let mut txn = index.env.write_txn().unwrap();
let rng = rand::rngs::SmallRng::from_seed([0; 32]); let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
let keys = let keys =
std::iter::from_fn(|| Some(rng.gen_range(0..256))).take(128).collect::<Vec<u32>>(); std::iter::from_fn(|| Some(rng.gen_range(0..256))).take(128).collect::<Vec<u32>>();
for (_i, key) in keys.into_iter().enumerate() { for (_i, key) in keys.into_iter().enumerate() {
let mut bitmap = RoaringBitmap::new(); let mut bitmap = RoaringBitmap::new();
bitmap.insert(key); bitmap.insert(key);
bitmap.insert(key + 100.); bitmap.insert(key + 100);
index.insert(&mut txn, 0, &(key as f64), &bitmap); index.insert(&mut txn, 0, &(key as f64), &bitmap);
} }
txn.commit().unwrap(); txn.commit().unwrap();
@ -160,15 +158,15 @@ mod tests {
#[test] #[test]
fn filter_sort_descending() { fn filter_sort_descending() {
let indexes = [get_simple_index(), get_random_looking_index()]; let indexes = [get_simple_index(), get_random_looking_index()];
for (i, index) in indexes.into_iter().enumerate() { for (i, index) in indexes.iter().enumerate() {
let txn = index.env.read_txn().unwrap(); let txn = index.env.read_txn().unwrap();
let candidates = (200..=300).into_iter().collect::<RoaringBitmap>(); let candidates = (200..=300).into_iter().collect::<RoaringBitmap>();
let mut results = String::new(); let mut results = String::new();
let db = index.db.content.remap_key_type::<FacetKeyCodec<MyByteSlice>>(); let db = index.db.content.remap_key_type::<FacetKeyCodec<MyByteSlice>>();
let iter = descending_facet_sort(&txn, &db, 0, candidates); let iter = descending_facet_sort(&txn, db, 0, candidates).unwrap();
for (facet, docids) in iter { for el in iter {
let facet = OrderedF64Codec::bytes_decode(facet).unwrap(); let docids = el.unwrap();
results.push_str(&format!("{facet}: {}\n", display_bitmap(&docids))); results.push_str(&display_bitmap(&docids));
} }
insta::assert_snapshot!(format!("filter_sort_{i}_descending"), results); insta::assert_snapshot!(format!("filter_sort_{i}_descending"), results);

16
milli/src/search/facet/mod.rs
View File

@ -78,9 +78,10 @@ pub(crate) fn get_highest_level<'t>(
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use crate::update::FacetsUpdateIncremental;
use heed::{BytesDecode, BytesEncode, Env, RwTxn};
use roaring::RoaringBitmap;
use std::{fmt::Display, marker::PhantomData, rc::Rc}; use std::{fmt::Display, marker::PhantomData, rc::Rc};
use heed::{BytesDecode, BytesEncode, Env};
use tempfile::TempDir; use tempfile::TempDir;
use crate::{ use crate::{
@ -148,6 +149,17 @@ mod test {
_phantom: PhantomData, _phantom: PhantomData,
} }
} }
pub fn insert<'a>(
&self,
rwtxn: &'a mut RwTxn,
field_id: u16,
key: &'a <BoundCodec as BytesEncode<'a>>::EItem,
docids: &RoaringBitmap,
) {
let update = FacetsUpdateIncremental::new(self.db.content);
let key_bytes = BoundCodec::bytes_encode(&key).unwrap();
update.insert(rwtxn, field_id, &key_bytes, docids).unwrap();
}
} }
impl<BoundCodec> Display for FacetIndex<BoundCodec> impl<BoundCodec> Display for FacetIndex<BoundCodec>

11
milli/src/snapshot_tests.rs
View File

@ -1,15 +1,10 @@
use crate::heed_codec::facet::new::{FacetGroupValue, FacetKey};
use crate::{make_db_snap_from_iter, ExternalDocumentsIds, Index};
use roaring::RoaringBitmap;
use std::borrow::Cow; use std::borrow::Cow;
use std::fmt::Write; use std::fmt::Write;
use std::path::Path; use std::path::Path;
use heed::types::ByteSlice;
use heed::BytesDecode;
use roaring::RoaringBitmap;
use crate::heed_codec::facet::new::{FacetGroupValue, FacetKey};
use crate::heed_codec::facet::FacetStringZeroBoundsValueCodec;
use crate::{make_db_snap_from_iter, CboRoaringBitmapCodec, ExternalDocumentsIds, Index};
#[track_caller] #[track_caller]
pub fn default_db_snapshot_settings_for_test(name: Option<&str>) -> insta::Settings { pub fn default_db_snapshot_settings_for_test(name: Option<&str>) -> insta::Settings {
let mut settings = insta::Settings::clone_current(); let mut settings = insta::Settings::clone_current();

13
milli/src/update/facet/incremental.rs
View File

@ -17,13 +17,18 @@ enum DeletionResult {
Remove { prev: Option<Vec<u8>>, next: Option<Vec<u8>> }, Remove { prev: Option<Vec<u8>>, next: Option<Vec<u8>> },
} }
struct FacetUpdateIncremental { pub struct FacetsUpdateIncremental {
db: heed::Database<FacetKeyCodec<MyByteSlice>, FacetGroupValueCodec>, db: heed::Database<FacetKeyCodec<MyByteSlice>, FacetGroupValueCodec>,
group_size: usize, group_size: usize,
min_level_size: usize, min_level_size: usize,
max_group_size: usize, max_group_size: usize,
} }
impl FacetUpdateIncremental { impl FacetsUpdateIncremental {
pub fn new(db: heed::Database<FacetKeyCodec<MyByteSlice>, FacetGroupValueCodec>) -> Self {
Self { db, group_size: 4, min_level_size: 5, max_group_size: 8 }
}
}
impl FacetsUpdateIncremental {
fn find_insertion_key_value( fn find_insertion_key_value(
&self, &self,
field_id: u16, field_id: u16,
@ -263,7 +268,7 @@ impl FacetUpdateIncremental {
} }
let group_size = self.group_size; let group_size = self.group_size;
let highest_level = get_highest_level(&txn, *self.db, field_id)?; let highest_level = get_highest_level(&txn, self.db, field_id)?;
let result = let result =
self.insert_in_level(txn, field_id, highest_level as u8, new_key, new_values)?; self.insert_in_level(txn, field_id, highest_level as u8, new_key, new_values)?;
@ -412,7 +417,7 @@ impl FacetUpdateIncremental {
if self.db.get(txn, &FacetKey { field_id, level: 0, left_bound: key })?.is_none() { if self.db.get(txn, &FacetKey { field_id, level: 0, left_bound: key })?.is_none() {
return Ok(()); return Ok(());
} }
let highest_level = get_highest_level(&txn, *self.db, field_id)?; let highest_level = get_highest_level(&txn, self.db, field_id)?;
// let key_bytes = BoundCodec::bytes_encode(&key).unwrap(); // let key_bytes = BoundCodec::bytes_encode(&key).unwrap();

1
milli/src/update/mod.rs
View File

@ -2,6 +2,7 @@ pub use self::available_documents_ids::AvailableDocumentsIds;
pub use self::clear_documents::ClearDocuments; pub use self::clear_documents::ClearDocuments;
pub use self::delete_documents::{DeleteDocuments, DocumentDeletionResult}; pub use self::delete_documents::{DeleteDocuments, DocumentDeletionResult};
pub use self::facet::bulk::FacetsUpdateBulk; pub use self::facet::bulk::FacetsUpdateBulk;
pub use self::facet::incremental::FacetsUpdateIncremental;
pub use self::index_documents::{ pub use self::index_documents::{
DocumentAdditionResult, DocumentId, IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, DocumentAdditionResult, DocumentId, IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod,
}; };