diff --git a/milli/src/update/facets.rs b/milli/src/update/facets.rs index a2f17cba3..19684c6ea 100644 --- a/milli/src/update/facets.rs +++ b/milli/src/update/facets.rs @@ -15,9 +15,7 @@ use crate::heed_codec::facet::{ FacetStringLevelZeroValueCodec, FacetStringZeroBoundsValueCodec, }; use crate::heed_codec::CboRoaringBitmapCodec; -use crate::update::index_documents::{ - create_writer, write_into_lmdb_database, writer_into_reader, WriteMethod, -}; +use crate::update::index_documents::{create_writer, write_into_lmdb_database, writer_into_reader}; use crate::{FieldId, Index, Result}; pub struct Facets<'t, 'u, 'i> { @@ -120,7 +118,6 @@ impl<'t, 'u, 'i> Facets<'t, 'u, 'i> { *self.index.facet_id_f64_docids.as_polymorph(), facet_number_levels, |_, _| Err(InternalError::IndexingMergingKeys { process: "facet number levels" })?, - WriteMethod::GetMergePut, )?; write_into_lmdb_database( @@ -128,7 +125,6 @@ impl<'t, 'u, 'i> Facets<'t, 'u, 'i> { *self.index.facet_id_string_docids.as_polymorph(), facet_string_levels, |_, _| Err(InternalError::IndexingMergingKeys { process: "facet string levels" })?, - WriteMethod::GetMergePut, )?; } diff --git a/milli/src/update/index_documents/helpers/grenad_helpers.rs b/milli/src/update/index_documents/helpers/grenad_helpers.rs index 10662892b..eef067122 100644 --- a/milli/src/update/index_documents/helpers/grenad_helpers.rs +++ b/milli/src/update/index_documents/helpers/grenad_helpers.rs @@ -9,7 +9,6 @@ use log::debug; use super::{ClonableMmap, MergeFn}; use crate::error::InternalError; -use crate::update::index_documents::WriteMethod; use crate::Result; pub type CursorClonableMmap = io::Cursor; @@ -169,34 +168,22 @@ pub fn write_into_lmdb_database( database: heed::PolyDatabase, mut reader: Reader, merge: MergeFn, - method: WriteMethod, ) -> Result<()> { debug!("Writing MTBL stores..."); let before = Instant::now(); - match method { - WriteMethod::Append => { - let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?; - while let Some((k, v)) = reader.next()? { + while let Some((k, v)) = reader.next()? { + let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?; + match iter.next().transpose()? { + Some((key, old_val)) if key == k => { + let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..]; + let val = merge(k, &vals)?; // safety: we don't keep references from inside the LMDB database. - unsafe { out_iter.append(k, v)? }; + unsafe { iter.put_current(k, &val)? }; } - } - WriteMethod::GetMergePut => { - while let Some((k, v)) = reader.next()? { - let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?; - match iter.next().transpose()? { - Some((key, old_val)) if key == k => { - let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..]; - let val = merge(k, &vals)?; - // safety: we don't keep references from inside the LMDB database. - unsafe { iter.put_current(k, &val)? }; - } - _ => { - drop(iter); - database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?; - } - } + _ => { + drop(iter); + database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?; } } } @@ -210,12 +197,11 @@ pub fn sorter_into_lmdb_database( database: heed::PolyDatabase, sorter: Sorter, merge: MergeFn, - method: WriteMethod, ) -> Result<()> { debug!("Writing MTBL sorter..."); let before = Instant::now(); - merger_iter_into_lmdb_database(wtxn, database, sorter.into_merger_iter()?, merge, method)?; + merger_iter_into_lmdb_database(wtxn, database, sorter.into_merger_iter()?, merge)?; debug!("MTBL sorter writen in {:.02?}!", before.elapsed()); Ok(()) @@ -226,34 +212,22 @@ fn merger_iter_into_lmdb_database( database: heed::PolyDatabase, mut sorter: MergerIter, merge: MergeFn, - method: WriteMethod, ) -> Result<()> { - match method { - WriteMethod::Append => { - let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?; - while let Some((k, v)) = sorter.next()? { + while let Some((k, v)) = sorter.next()? { + let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?; + match iter.next().transpose()? { + Some((key, old_val)) if key == k => { + let vals = vec![Cow::Borrowed(old_val), Cow::Borrowed(v)]; + let val = merge(k, &vals).map_err(|_| { + // TODO just wrap this error? + InternalError::IndexingMergingKeys { process: "get-put-merge" } + })?; // safety: we don't keep references from inside the LMDB database. - unsafe { out_iter.append(k, v)? }; + unsafe { iter.put_current(k, &val)? }; } - } - WriteMethod::GetMergePut => { - while let Some((k, v)) = sorter.next()? { - let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?; - match iter.next().transpose()? { - Some((key, old_val)) if key == k => { - let vals = vec![Cow::Borrowed(old_val), Cow::Borrowed(v)]; - let val = merge(k, &vals).map_err(|_| { - // TODO just wrap this error? - InternalError::IndexingMergingKeys { process: "get-put-merge" } - })?; - // safety: we don't keep references from inside the LMDB database. - unsafe { iter.put_current(k, &val)? }; - } - _ => { - drop(iter); - database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?; - } - } + _ => { + drop(iter); + database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?; } } } diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 7ea5c3816..ee80d8ada 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -59,12 +59,6 @@ impl Default for IndexDocumentsMethod { } } -#[derive(Debug, Copy, Clone)] -pub enum WriteMethod { - Append, - GetMergePut, -} - pub struct IndexDocuments<'t, 'u, 'i, 'a, F> { wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index, diff --git a/milli/src/update/word_prefix_docids.rs b/milli/src/update/word_prefix_docids.rs index 1e2996c9b..cf50a5b8a 100644 --- a/milli/src/update/word_prefix_docids.rs +++ b/milli/src/update/word_prefix_docids.rs @@ -7,7 +7,7 @@ use slice_group_by::GroupBy; use crate::update::index_documents::{ create_sorter, fst_stream_into_hashset, fst_stream_into_vec, merge_roaring_bitmaps, - sorter_into_lmdb_database, CursorClonableMmap, MergeFn, WriteMethod, + sorter_into_lmdb_database, CursorClonableMmap, MergeFn, }; use crate::{Index, Result}; @@ -126,7 +126,6 @@ impl<'t, 'u, 'i> WordPrefixDocids<'t, 'u, 'i> { *self.index.word_prefix_docids.as_polymorph(), prefix_docids_sorter, merge_roaring_bitmaps, - WriteMethod::GetMergePut, )?; Ok(()) diff --git a/milli/src/update/word_prefix_pair_proximity_docids.rs b/milli/src/update/word_prefix_pair_proximity_docids.rs index f846e8d9e..5b025e4fc 100644 --- a/milli/src/update/word_prefix_pair_proximity_docids.rs +++ b/milli/src/update/word_prefix_pair_proximity_docids.rs @@ -8,7 +8,7 @@ use slice_group_by::GroupBy; use crate::update::index_documents::{ create_sorter, fst_stream_into_hashset, fst_stream_into_vec, merge_cbo_roaring_bitmaps, - sorter_into_lmdb_database, CursorClonableMmap, MergeFn, WriteMethod, + sorter_into_lmdb_database, CursorClonableMmap, MergeFn, }; use crate::{Index, Result, StrStrU8Codec}; @@ -192,7 +192,6 @@ impl<'t, 'u, 'i> WordPrefixPairProximityDocids<'t, 'u, 'i> { *self.index.word_prefix_pair_proximity_docids.as_polymorph(), word_prefix_pair_proximity_docids_sorter, merge_cbo_roaring_bitmaps, - WriteMethod::GetMergePut, )?; Ok(()) diff --git a/milli/src/update/words_prefix_position_docids.rs b/milli/src/update/words_prefix_position_docids.rs index b1b8273ef..178684cf0 100644 --- a/milli/src/update/words_prefix_position_docids.rs +++ b/milli/src/update/words_prefix_position_docids.rs @@ -14,7 +14,7 @@ use crate::heed_codec::StrBEU32Codec; use crate::index::main_key::WORDS_PREFIXES_FST_KEY; use crate::update::index_documents::{ create_sorter, fst_stream_into_hashset, fst_stream_into_vec, merge_cbo_roaring_bitmaps, - sorter_into_lmdb_database, CursorClonableMmap, MergeFn, WriteMethod, + sorter_into_lmdb_database, CursorClonableMmap, MergeFn, }; use crate::{Index, Result}; @@ -170,7 +170,6 @@ impl<'t, 'u, 'i> WordPrefixPositionDocids<'t, 'u, 'i> { *self.index.word_prefix_position_docids.as_polymorph(), prefix_position_docids_sorter, merge_cbo_roaring_bitmaps, - WriteMethod::GetMergePut, )?; Ok(())