Move the fst_stream_into_hashset method in the helper methods

milli/src/update/index_documents/helpers/mod.rs

@@ -2,9 +2,11 @@ mod clonable_mmap;
 mod grenad_helpers;
 mod merge_functions;
 
+use std::collections::HashSet;
 use std::convert::{TryFrom, TryInto};
 
 pub use clonable_mmap::{ClonableMmap, CursorClonableMmap};
+use fst::{IntoStreamer, Streamer};
 pub use grenad_helpers::{
     create_sorter, create_writer, grenad_obkv_into_chunks, into_clonable_grenad, merge_readers,
     sorter_into_lmdb_database, sorter_into_reader, write_into_lmdb_database, writer_into_reader,
@@ -43,3 +45,17 @@ where
 pub fn read_u32_ne_bytes(bytes: &[u8]) -> impl Iterator<Item = u32> + '_ {
     bytes.chunks_exact(4).flat_map(TryInto::try_into).map(u32::from_ne_bytes)
 }
+
+/// Converts an fst Stream into an HashSet.
+pub fn fst_stream_into_hashset<'f, I, S>(stream: I) -> HashSet<Vec<u8>>
+where
+    I: for<'a> IntoStreamer<'a, Into = S, Item = &'a [u8]>,
+    S: 'f + for<'a> Streamer<'a, Item = &'a [u8]>,
+{
+    let mut hashset = HashSet::new();
+    let mut stream = stream.into_stream();
+    while let Some(value) = stream.next() {
+        hashset.insert(value.to_owned());
+    }
+    hashset
+}

milli/src/update/index_documents/mod.rs

@@ -15,8 +15,9 @@ use serde::{Deserialize, Serialize};
 use typed_chunk::{write_typed_chunk_into_index, TypedChunk};
 
 pub use self::helpers::{
-    create_sorter, create_writer, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps,
+    create_sorter, create_writer, fst_stream_into_hashset, merge_cbo_roaring_bitmaps,
-    sorter_into_lmdb_database, write_into_lmdb_database, writer_into_reader, ClonableMmap, MergeFn,
+    merge_roaring_bitmaps, sorter_into_lmdb_database, write_into_lmdb_database, writer_into_reader,
+    ClonableMmap, MergeFn,
 };
 use self::helpers::{grenad_obkv_into_chunks, GrenadParameters};
 pub use self::transform::{Transform, TransformOutput};

milli/src/update/word_prefix_pair_proximity_docids.rs

@@ -1,14 +1,14 @@
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 
-use fst::{IntoStreamer, Streamer};
+use fst::IntoStreamer;
 use grenad::{CompressionType, MergerBuilder};
 use heed::BytesDecode;
 use log::debug;
 use slice_group_by::GroupBy;
 
 use crate::update::index_documents::{
-    create_sorter, merge_cbo_roaring_bitmaps, sorter_into_lmdb_database, CursorClonableMmap,
+    create_sorter, fst_stream_into_hashset, merge_cbo_roaring_bitmaps, sorter_into_lmdb_database,
-    MergeFn, WriteMethod,
+    CursorClonableMmap, MergeFn, WriteMethod,
 };
 use crate::{Index, Result, StrStrU8Codec};
 
@@ -89,7 +89,7 @@ impl<'t, 'u, 'i> WordPrefixPairProximityDocids<'t, 'u, 'i> {
             prefix_fst_keys.as_slice().linear_group_by_key(|x| x.chars().nth(0).unwrap()).collect();
 
         // We compute the set of prefixes that are no more part of the prefix fst.
-        let suppr_pw = stream_into_hashset(old_prefix_fst.op().add(&prefix_fst).difference());
+        let suppr_pw = fst_stream_into_hashset(old_prefix_fst.op().add(&prefix_fst).difference());
 
         let mut buffer = Vec::new();
         let mut current_prefixes: Option<&&[String]> = None;
@@ -178,17 +178,3 @@ fn write_prefixes_in_sorter(
 
     Ok(())
 }
-
-/// Converts an fst Stream into an HashSet.
-fn stream_into_hashset<'f, I, S>(stream: I) -> HashSet<Vec<u8>>
-where
-    I: for<'a> IntoStreamer<'a, Into = S, Item = &'a [u8]>,
-    S: 'f + for<'a> Streamer<'a, Item = &'a [u8]>,
-{
-    let mut hashset = HashSet::new();
-    let mut stream = stream.into_stream();
-    while let Some(value) = stream.next() {
-        hashset.insert(value.to_owned());
-    }
-    hashset
-}