From 5404bc02dddaf790b23d71cb1834b53a461c06fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <clement@meilisearch.com>
Date: Wed, 19 Jan 2022 14:30:03 +0100
Subject: [PATCH] Move the fst_stream_into_hashset method in the helper methods

---
 .../src/update/index_documents/helpers/mod.rs | 16 +++++++++++++
 milli/src/update/index_documents/mod.rs       |  5 ++--
 .../word_prefix_pair_proximity_docids.rs      | 24 ++++---------------
 3 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/milli/src/update/index_documents/helpers/mod.rs b/milli/src/update/index_documents/helpers/mod.rs
index 128288982..4086bfb7f 100644
--- a/milli/src/update/index_documents/helpers/mod.rs
+++ b/milli/src/update/index_documents/helpers/mod.rs
@@ -2,9 +2,11 @@ mod clonable_mmap;
 mod grenad_helpers;
 mod merge_functions;
 
+use std::collections::HashSet;
 use std::convert::{TryFrom, TryInto};
 
 pub use clonable_mmap::{ClonableMmap, CursorClonableMmap};
+use fst::{IntoStreamer, Streamer};
 pub use grenad_helpers::{
     create_sorter, create_writer, grenad_obkv_into_chunks, into_clonable_grenad, merge_readers,
     sorter_into_lmdb_database, sorter_into_reader, write_into_lmdb_database, writer_into_reader,
@@ -43,3 +45,17 @@ where
 pub fn read_u32_ne_bytes(bytes: &[u8]) -> impl Iterator<Item = u32> + '_ {
     bytes.chunks_exact(4).flat_map(TryInto::try_into).map(u32::from_ne_bytes)
 }
+
+/// Converts an fst Stream into an HashSet.
+pub fn fst_stream_into_hashset<'f, I, S>(stream: I) -> HashSet<Vec<u8>>
+where
+    I: for<'a> IntoStreamer<'a, Into = S, Item = &'a [u8]>,
+    S: 'f + for<'a> Streamer<'a, Item = &'a [u8]>,
+{
+    let mut hashset = HashSet::new();
+    let mut stream = stream.into_stream();
+    while let Some(value) = stream.next() {
+        hashset.insert(value.to_owned());
+    }
+    hashset
+}
diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs
index 0ef05dba5..ae8e28b33 100644
--- a/milli/src/update/index_documents/mod.rs
+++ b/milli/src/update/index_documents/mod.rs
@@ -15,8 +15,9 @@ use serde::{Deserialize, Serialize};
 use typed_chunk::{write_typed_chunk_into_index, TypedChunk};
 
 pub use self::helpers::{
-    create_sorter, create_writer, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps,
-    sorter_into_lmdb_database, write_into_lmdb_database, writer_into_reader, ClonableMmap, MergeFn,
+    create_sorter, create_writer, fst_stream_into_hashset, merge_cbo_roaring_bitmaps,
+    merge_roaring_bitmaps, sorter_into_lmdb_database, write_into_lmdb_database, writer_into_reader,
+    ClonableMmap, MergeFn,
 };
 use self::helpers::{grenad_obkv_into_chunks, GrenadParameters};
 pub use self::transform::{Transform, TransformOutput};
diff --git a/milli/src/update/word_prefix_pair_proximity_docids.rs b/milli/src/update/word_prefix_pair_proximity_docids.rs
index d35f39c10..50b7a978b 100644
--- a/milli/src/update/word_prefix_pair_proximity_docids.rs
+++ b/milli/src/update/word_prefix_pair_proximity_docids.rs
@@ -1,14 +1,14 @@
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 
-use fst::{IntoStreamer, Streamer};
+use fst::IntoStreamer;
 use grenad::{CompressionType, MergerBuilder};
 use heed::BytesDecode;
 use log::debug;
 use slice_group_by::GroupBy;
 
 use crate::update::index_documents::{
-    create_sorter, merge_cbo_roaring_bitmaps, sorter_into_lmdb_database, CursorClonableMmap,
-    MergeFn, WriteMethod,
+    create_sorter, fst_stream_into_hashset, merge_cbo_roaring_bitmaps, sorter_into_lmdb_database,
+    CursorClonableMmap, MergeFn, WriteMethod,
 };
 use crate::{Index, Result, StrStrU8Codec};
 
@@ -89,7 +89,7 @@ impl<'t, 'u, 'i> WordPrefixPairProximityDocids<'t, 'u, 'i> {
             prefix_fst_keys.as_slice().linear_group_by_key(|x| x.chars().nth(0).unwrap()).collect();
 
         // We compute the set of prefixes that are no more part of the prefix fst.
-        let suppr_pw = stream_into_hashset(old_prefix_fst.op().add(&prefix_fst).difference());
+        let suppr_pw = fst_stream_into_hashset(old_prefix_fst.op().add(&prefix_fst).difference());
 
         let mut buffer = Vec::new();
         let mut current_prefixes: Option<&&[String]> = None;
@@ -178,17 +178,3 @@ fn write_prefixes_in_sorter(
 
     Ok(())
 }
-
-/// Converts an fst Stream into an HashSet.
-fn stream_into_hashset<'f, I, S>(stream: I) -> HashSet<Vec<u8>>
-where
-    I: for<'a> IntoStreamer<'a, Into = S, Item = &'a [u8]>,
-    S: 'f + for<'a> Streamer<'a, Item = &'a [u8]>,
-{
-    let mut hashset = HashSet::new();
-    let mut stream = stream.into_stream();
-    while let Some(value) = stream.next() {
-        hashset.insert(value.to_owned());
-    }
-    hashset
-}