diff --git a/milli/src/update/index_documents/helpers/mod.rs b/milli/src/update/index_documents/helpers/mod.rs index 128288982..4086bfb7f 100644 --- a/milli/src/update/index_documents/helpers/mod.rs +++ b/milli/src/update/index_documents/helpers/mod.rs @@ -2,9 +2,11 @@ mod clonable_mmap; mod grenad_helpers; mod merge_functions; +use std::collections::HashSet; use std::convert::{TryFrom, TryInto}; pub use clonable_mmap::{ClonableMmap, CursorClonableMmap}; +use fst::{IntoStreamer, Streamer}; pub use grenad_helpers::{ create_sorter, create_writer, grenad_obkv_into_chunks, into_clonable_grenad, merge_readers, sorter_into_lmdb_database, sorter_into_reader, write_into_lmdb_database, writer_into_reader, @@ -43,3 +45,17 @@ where pub fn read_u32_ne_bytes(bytes: &[u8]) -> impl Iterator + '_ { bytes.chunks_exact(4).flat_map(TryInto::try_into).map(u32::from_ne_bytes) } + +/// Converts an fst Stream into an HashSet. +pub fn fst_stream_into_hashset<'f, I, S>(stream: I) -> HashSet> +where + I: for<'a> IntoStreamer<'a, Into = S, Item = &'a [u8]>, + S: 'f + for<'a> Streamer<'a, Item = &'a [u8]>, +{ + let mut hashset = HashSet::new(); + let mut stream = stream.into_stream(); + while let Some(value) = stream.next() { + hashset.insert(value.to_owned()); + } + hashset +} diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 0ef05dba5..ae8e28b33 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -15,8 +15,9 @@ use serde::{Deserialize, Serialize}; use typed_chunk::{write_typed_chunk_into_index, TypedChunk}; pub use self::helpers::{ - create_sorter, create_writer, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, - sorter_into_lmdb_database, write_into_lmdb_database, writer_into_reader, ClonableMmap, MergeFn, + create_sorter, create_writer, fst_stream_into_hashset, merge_cbo_roaring_bitmaps, + merge_roaring_bitmaps, sorter_into_lmdb_database, write_into_lmdb_database, writer_into_reader, + ClonableMmap, MergeFn, }; use self::helpers::{grenad_obkv_into_chunks, GrenadParameters}; pub use self::transform::{Transform, TransformOutput}; diff --git a/milli/src/update/word_prefix_pair_proximity_docids.rs b/milli/src/update/word_prefix_pair_proximity_docids.rs index d35f39c10..50b7a978b 100644 --- a/milli/src/update/word_prefix_pair_proximity_docids.rs +++ b/milli/src/update/word_prefix_pair_proximity_docids.rs @@ -1,14 +1,14 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; -use fst::{IntoStreamer, Streamer}; +use fst::IntoStreamer; use grenad::{CompressionType, MergerBuilder}; use heed::BytesDecode; use log::debug; use slice_group_by::GroupBy; use crate::update::index_documents::{ - create_sorter, merge_cbo_roaring_bitmaps, sorter_into_lmdb_database, CursorClonableMmap, - MergeFn, WriteMethod, + create_sorter, fst_stream_into_hashset, merge_cbo_roaring_bitmaps, sorter_into_lmdb_database, + CursorClonableMmap, MergeFn, WriteMethod, }; use crate::{Index, Result, StrStrU8Codec}; @@ -89,7 +89,7 @@ impl<'t, 'u, 'i> WordPrefixPairProximityDocids<'t, 'u, 'i> { prefix_fst_keys.as_slice().linear_group_by_key(|x| x.chars().nth(0).unwrap()).collect(); // We compute the set of prefixes that are no more part of the prefix fst. - let suppr_pw = stream_into_hashset(old_prefix_fst.op().add(&prefix_fst).difference()); + let suppr_pw = fst_stream_into_hashset(old_prefix_fst.op().add(&prefix_fst).difference()); let mut buffer = Vec::new(); let mut current_prefixes: Option<&&[String]> = None; @@ -178,17 +178,3 @@ fn write_prefixes_in_sorter( Ok(()) } - -/// Converts an fst Stream into an HashSet. -fn stream_into_hashset<'f, I, S>(stream: I) -> HashSet> -where - I: for<'a> IntoStreamer<'a, Into = S, Item = &'a [u8]>, - S: 'f + for<'a> Streamer<'a, Item = &'a [u8]>, -{ - let mut hashset = HashSet::new(); - let mut stream = stream.into_stream(); - while let Some(value) = stream.next() { - hashset.insert(value.to_owned()); - } - hashset -}