From d3575fb0280cb5a37cd17d1a904026080092ffe4 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Mon, 20 Nov 2023 10:53:40 +0100 Subject: [PATCH] Make into_del_add_obkv parameters more human readable --- milli/src/update/del_add.rs | 25 ++++--- milli/src/update/index_documents/transform.rs | 75 +++++++++++++------ 2 files changed, 68 insertions(+), 32 deletions(-) diff --git a/milli/src/update/del_add.rs b/milli/src/update/del_add.rs index 07a20b025..794beb5df 100644 --- a/milli/src/update/del_add.rs +++ b/milli/src/update/del_add.rs @@ -32,13 +32,12 @@ impl Key for DelAdd { /// Creates a Kv> from Kv /// -/// if deletion is `true`, the value will be inserted behind a DelAdd::Deletion key. -/// if addition is `true`, the value will be inserted behind a DelAdd::Addition key. -/// if both deletion and addition are `true, the value will be inserted in both keys. +/// Deletion: put all the values under DelAdd::Deletion +/// Addition: put all the values under DelAdd::Addition, +/// DeletionAndAddition: put all the values under DelAdd::Deletion and DelAdd::Addition, pub fn into_del_add_obkv( reader: obkv::KvReader, - deletion: bool, - addition: bool, + operation: DelAddOperation, buffer: &mut Vec, ) -> Result<(), std::io::Error> { let mut writer = obkv::KvWriter::new(buffer); @@ -46,21 +45,27 @@ pub fn into_del_add_obkv( for (key, value) in reader.iter() { value_buffer.clear(); let mut value_writer = KvWriterDelAdd::new(&mut value_buffer); - if deletion { + if matches!(operation, DelAddOperation::Deletion | DelAddOperation::DeletionAndAddition) { value_writer.insert(DelAdd::Deletion, value)?; } - if addition { + if matches!(operation, DelAddOperation::Addition | DelAddOperation::DeletionAndAddition) { value_writer.insert(DelAdd::Addition, value)?; } value_writer.finish()?; - if !value_buffer.is_empty() { - writer.insert(key, &value_buffer)?; - } + writer.insert(key, &value_buffer)?; } writer.finish() } +/// Enum controlling the side of the DelAdd obkv in which the provided value will be written. +#[derive(Debug, Clone, Copy)] +pub enum DelAddOperation { + Deletion, + Addition, + DeletionAndAddition, +} + /// Creates a Kv> from two Kv /// /// putting each deletion obkv's keys under an DelAdd::Deletion diff --git a/milli/src/update/index_documents/transform.rs b/milli/src/update/index_documents/transform.rs index 8dc88efb9..323bc3da7 100644 --- a/milli/src/update/index_documents/transform.rs +++ b/milli/src/update/index_documents/transform.rs @@ -21,7 +21,7 @@ use super::{IndexDocumentsMethod, IndexerConfig}; use crate::documents::{DocumentsBatchIndex, EnrichedDocument, EnrichedDocumentsBatchReader}; use crate::error::{Error, InternalError, UserError}; use crate::index::{db_name, main_key}; -use crate::update::del_add::{into_del_add_obkv, DelAdd, KvReaderDelAdd}; +use crate::update::del_add::{into_del_add_obkv, DelAdd, DelAddOperation, KvReaderDelAdd}; use crate::update::index_documents::GrenadParameters; use crate::update::{AvailableDocumentsIds, ClearDocuments, UpdateIndexingStep}; use crate::{ @@ -265,8 +265,12 @@ impl<'a, 'i> Transform<'a, 'i> { skip_insertion = true; } else { // we associate the base document with the new key, everything will get merged later. - let keep_original_version = - self.index_documents_method == IndexDocumentsMethod::UpdateDocuments; + let deladd_operation = match self.index_documents_method { + IndexDocumentsMethod::UpdateDocuments => { + DelAddOperation::DeletionAndAddition + } + IndexDocumentsMethod::ReplaceDocuments => DelAddOperation::Deletion, + }; document_sorter_key_buffer.clear(); document_sorter_key_buffer.extend_from_slice(&docid.to_be_bytes()); document_sorter_key_buffer.extend_from_slice(external_id.as_bytes()); @@ -274,8 +278,7 @@ impl<'a, 'i> Transform<'a, 'i> { document_sorter_value_buffer.push(Operation::Addition as u8); into_del_add_obkv( KvReaderU16::new(base_obkv), - true, - keep_original_version, + deladd_operation, &mut document_sorter_value_buffer, )?; self.original_sorter @@ -287,8 +290,7 @@ impl<'a, 'i> Transform<'a, 'i> { document_sorter_value_buffer.push(Operation::Addition as u8); into_del_add_obkv( KvReaderU16::new(&flattened_obkv), - true, - keep_original_version, + deladd_operation, &mut document_sorter_value_buffer, )?; } @@ -307,8 +309,7 @@ impl<'a, 'i> Transform<'a, 'i> { document_sorter_value_buffer.push(Operation::Addition as u8); into_del_add_obkv( KvReaderU16::new(&obkv_buffer), - false, - true, + DelAddOperation::Addition, &mut document_sorter_value_buffer, )?; // We use the extracted/generated user id as the key for this document. @@ -321,8 +322,7 @@ impl<'a, 'i> Transform<'a, 'i> { document_sorter_value_buffer.push(Operation::Addition as u8); into_del_add_obkv( KvReaderU16::new(&obkv), - false, - true, + DelAddOperation::Addition, &mut document_sorter_value_buffer, )? } @@ -517,7 +517,11 @@ impl<'a, 'i> Transform<'a, 'i> { // push it as to delete in the original_sorter document_sorter_value_buffer.clear(); document_sorter_value_buffer.push(Operation::Deletion as u8); - into_del_add_obkv(KvReaderU16::new(base_obkv), true, false, document_sorter_value_buffer)?; + into_del_add_obkv( + KvReaderU16::new(base_obkv), + DelAddOperation::Deletion, + document_sorter_value_buffer, + )?; self.original_sorter.insert(&document_sorter_key_buffer, &document_sorter_value_buffer)?; // flatten it and push it as to delete in the flattened_sorter @@ -526,7 +530,11 @@ impl<'a, 'i> Transform<'a, 'i> { // we recreate our buffer with the flattened documents document_sorter_value_buffer.clear(); document_sorter_value_buffer.push(Operation::Deletion as u8); - into_del_add_obkv(KvReaderU16::new(&obkv), true, false, document_sorter_value_buffer)?; + into_del_add_obkv( + KvReaderU16::new(&obkv), + DelAddOperation::Deletion, + document_sorter_value_buffer, + )?; } self.flattened_sorter .insert(internal_docid.to_be_bytes(), &document_sorter_value_buffer)?; @@ -869,8 +877,7 @@ impl<'a, 'i> Transform<'a, 'i> { document_sorter_value_buffer.clear(); into_del_add_obkv( KvReaderU16::new(buffer), - false, - true, + DelAddOperation::Addition, &mut document_sorter_value_buffer, )?; original_sorter.insert(&document_sorter_key_buffer, &document_sorter_value_buffer)?; @@ -911,8 +918,7 @@ impl<'a, 'i> Transform<'a, 'i> { document_sorter_value_buffer.clear(); into_del_add_obkv( KvReaderU16::new(&buffer), - false, - true, + DelAddOperation::Addition, &mut document_sorter_value_buffer, )?; flattened_sorter.insert(docid.to_be_bytes(), &document_sorter_value_buffer)?; @@ -986,18 +992,38 @@ mod test { let mut kv_writer = KvWriter::memory(); kv_writer.insert(0_u8, [0]).unwrap(); let buffer = kv_writer.into_inner().unwrap(); - into_del_add_obkv(KvReaderU16::new(&buffer), false, true, &mut additive_doc_0).unwrap(); + into_del_add_obkv( + KvReaderU16::new(&buffer), + DelAddOperation::Addition, + &mut additive_doc_0, + ) + .unwrap(); additive_doc_0.insert(0, Operation::Addition as u8); - into_del_add_obkv(KvReaderU16::new(&buffer), true, false, &mut deletive_doc_0).unwrap(); + into_del_add_obkv( + KvReaderU16::new(&buffer), + DelAddOperation::Deletion, + &mut deletive_doc_0, + ) + .unwrap(); deletive_doc_0.insert(0, Operation::Deletion as u8); - into_del_add_obkv(KvReaderU16::new(&buffer), true, true, &mut del_add_doc_0).unwrap(); + into_del_add_obkv( + KvReaderU16::new(&buffer), + DelAddOperation::DeletionAndAddition, + &mut del_add_doc_0, + ) + .unwrap(); del_add_doc_0.insert(0, Operation::Addition as u8); let mut additive_doc_1 = Vec::new(); let mut kv_writer = KvWriter::memory(); kv_writer.insert(1_u8, [1]).unwrap(); let buffer = kv_writer.into_inner().unwrap(); - into_del_add_obkv(KvReaderU16::new(&buffer), false, true, &mut additive_doc_1).unwrap(); + into_del_add_obkv( + KvReaderU16::new(&buffer), + DelAddOperation::Addition, + &mut additive_doc_1, + ) + .unwrap(); additive_doc_1.insert(0, Operation::Addition as u8); let mut additive_doc_0_1 = Vec::new(); @@ -1005,7 +1031,12 @@ mod test { kv_writer.insert(0_u8, [0]).unwrap(); kv_writer.insert(1_u8, [1]).unwrap(); let buffer = kv_writer.into_inner().unwrap(); - into_del_add_obkv(KvReaderU16::new(&buffer), false, true, &mut additive_doc_0_1).unwrap(); + into_del_add_obkv( + KvReaderU16::new(&buffer), + DelAddOperation::Addition, + &mut additive_doc_0_1, + ) + .unwrap(); additive_doc_0_1.insert(0, Operation::Addition as u8); let ret = obkvs_merge_additions_and_deletions(&[], &[Cow::from(additive_doc_0.as_slice())])