From ac3baafbe85914a4020eae06f43f6b8394eff5ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Lecrenier?= Date: Wed, 16 Nov 2022 14:03:27 +0100 Subject: [PATCH] Truncate facet values that are too long before indexing them --- .../extract/extract_facet_string_docids.rs | 21 +++++++++++++------ .../extract/extract_fid_docid_facet_values.rs | 11 ++++++++-- .../src/update/index_documents/helpers/mod.rs | 14 ++++++++++++- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/milli/src/update/index_documents/extract/extract_facet_string_docids.rs b/milli/src/update/index_documents/extract/extract_facet_string_docids.rs index 8b02a6008..3a0af3c96 100644 --- a/milli/src/update/index_documents/extract/extract_facet_string_docids.rs +++ b/milli/src/update/index_documents/extract/extract_facet_string_docids.rs @@ -6,7 +6,8 @@ use heed::BytesEncode; use super::helpers::{create_sorter, sorter_into_reader, try_split_array_at, GrenadParameters}; use crate::heed_codec::facet::{FacetGroupKey, FacetGroupKeyCodec}; use crate::heed_codec::StrRefCodec; -use crate::update::index_documents::{merge_cbo_roaring_bitmaps, valid_lmdb_key}; +use crate::update::index_documents::helpers::MAX_FACET_VALUE_LENGTH; +use crate::update::index_documents::merge_cbo_roaring_bitmaps; use crate::{FieldId, Result}; /// Extracts the facet string and the documents ids where this facet string appear. @@ -38,13 +39,21 @@ pub fn extract_facet_string_docids( try_split_array_at::<_, 4>(bytes).unwrap(); let document_id = u32::from_be_bytes(document_id_bytes); - let normalised_value = std::str::from_utf8(normalized_value_bytes)?; + let mut normalised_value = std::str::from_utf8(normalized_value_bytes)?; + + let normalised_truncated_value: String; + if normalised_value.len() > MAX_FACET_VALUE_LENGTH { + normalised_truncated_value = normalised_value + .char_indices() + .take_while(|(idx, _)| idx + 4 < MAX_FACET_VALUE_LENGTH) + .map(|(_, c)| c) + .collect(); + normalised_value = normalised_truncated_value.as_str(); + } let key = FacetGroupKey { field_id, level: 0, left_bound: normalised_value }; let key_bytes = FacetGroupKeyCodec::::bytes_encode(&key).unwrap(); - if valid_lmdb_key(&key_bytes) { - // document id is encoded in native-endian because of the CBO roaring bitmap codec - facet_string_docids_sorter.insert(&key_bytes, document_id.to_ne_bytes())?; - } + // document id is encoded in native-endian because of the CBO roaring bitmap codec + facet_string_docids_sorter.insert(&key_bytes, document_id.to_ne_bytes())?; } sorter_into_reader(facet_string_docids_sorter, indexer) diff --git a/milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs b/milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs index 44afcde6c..b37cd90d3 100644 --- a/milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs +++ b/milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs @@ -12,6 +12,7 @@ use serde_json::Value; use super::helpers::{create_sorter, keep_first, sorter_into_reader, GrenadParameters}; use crate::error::InternalError; use crate::facet::value_encoding::f64_into_bytes; +use crate::update::index_documents::helpers::MAX_FACET_VALUE_LENGTH; use crate::update::index_documents::{create_writer, writer_into_reader}; use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result, BEU32}; @@ -85,10 +86,16 @@ pub fn extract_fid_docid_facet_values( } } - // insert normalized and original facet string in sorter + // insert normalized and original facet string in sorter for (normalized, original) in strings.into_iter().filter(|(n, _)| !n.is_empty()) { + let normalised_truncated_value: String = normalized + .char_indices() + .take_while(|(idx, _)| idx + 4 < MAX_FACET_VALUE_LENGTH) + .map(|(_, c)| c) + .collect(); + key_buffer.truncate(size_of::() + size_of::()); - key_buffer.extend_from_slice(normalized.as_bytes()); + key_buffer.extend_from_slice(normalised_truncated_value.as_bytes()); fid_docid_facet_strings_sorter.insert(&key_buffer, original.as_bytes())?; } } diff --git a/milli/src/update/index_documents/helpers/mod.rs b/milli/src/update/index_documents/helpers/mod.rs index 8fb629cae..e1f112858 100644 --- a/milli/src/update/index_documents/helpers/mod.rs +++ b/milli/src/update/index_documents/helpers/mod.rs @@ -18,8 +18,20 @@ pub use merge_functions::{ serialize_roaring_bitmap, MergeFn, }; +/// The maximum length a LMDB key can be. +/// +/// Note that the actual allowed length is a little bit higher, but +/// we keep a margin of safety. +const MAX_LMDB_KEY_LENGTH: usize = 500; + +/// The maximum length a field value can be when inserted in an LMDB key. +/// +/// This number is determined by the keys of the different facet databases +/// and adding a margin of safety. +pub const MAX_FACET_VALUE_LENGTH: usize = MAX_LMDB_KEY_LENGTH - 20; + /// The maximum length a word can be -pub const MAX_WORD_LENGTH: usize = 250; +pub const MAX_WORD_LENGTH: usize = MAX_LMDB_KEY_LENGTH / 2; pub fn valid_lmdb_key(key: impl AsRef<[u8]>) -> bool { key.as_ref().len() <= MAX_WORD_LENGTH * 2 && !key.as_ref().is_empty()