diff --git a/meilidb-data/src/database/documents_addition.rs b/meilidb-data/src/database/documents_addition.rs index 9f0794346..177d1975c 100644 --- a/meilidb-data/src/database/documents_addition.rs +++ b/meilidb-data/src/database/documents_addition.rs @@ -122,6 +122,7 @@ impl<'a> DocumentsAddition<'a> { let ranked_map = self.ranked_map; let schema = lease_inner.schema.clone(); let raw = lease_inner.raw.clone(); + lease_inner.raw.compact(); let inner = InnerIndex { words, schema, ranked_map, raw }; self.inner.0.store(Arc::new(inner)); diff --git a/meilidb-data/src/database/documents_deletion.rs b/meilidb-data/src/database/documents_deletion.rs index 46b423c7a..e89923199 100644 --- a/meilidb-data/src/database/documents_deletion.rs +++ b/meilidb-data/src/database/documents_deletion.rs @@ -121,6 +121,7 @@ impl<'a> DocumentsDeletion<'a> { let ranked_map = lease_inner.ranked_map.clone(); let schema = lease_inner.schema.clone(); let raw = lease_inner.raw.clone(); + lease_inner.raw.compact(); let inner = InnerIndex { words, schema, ranked_map, raw }; self.inner.0.store(Arc::new(inner)); diff --git a/meilidb-data/src/database/raw_index.rs b/meilidb-data/src/database/raw_index.rs index 793203a8d..8c129ac2d 100644 --- a/meilidb-data/src/database/raw_index.rs +++ b/meilidb-data/src/database/raw_index.rs @@ -10,6 +10,16 @@ pub struct RawIndex { pub custom: CustomSettings, } +impl RawIndex { + pub(crate) fn compact(&self) { + self.main.0.compact_range(None::<&[u8]>, None::<&[u8]>); + self.words.0.compact_range(None::<&[u8]>, None::<&[u8]>); + self.docs_words.0.compact_range(None::<&[u8]>, None::<&[u8]>); + self.documents.0.compact_range(None::<&[u8]>, None::<&[u8]>); + self.custom.0.compact_range(None::<&[u8]>, None::<&[u8]>); + } +} + #[derive(Clone)] pub struct InnerRawIndex { database: Arc, @@ -65,4 +75,12 @@ impl InnerRawIndex { self.database.write(batch) } + + pub fn compact_range(&self, start: Option, end: Option) + where S: AsRef<[u8]>, + E: AsRef<[u8]>, + { + let cf = self.database.cf_handle(&self.name).expect("cf not found"); + self.database.compact_range_cf(cf, start, end) + } } diff --git a/meilidb-data/src/database/words_index.rs b/meilidb-data/src/database/words_index.rs index 432a294e5..4f2163650 100644 --- a/meilidb-data/src/database/words_index.rs +++ b/meilidb-data/src/database/words_index.rs @@ -12,9 +12,24 @@ impl WordsIndex { // we must force an allocation to make the memory aligned match self.0.get(word)? { Some(bytes) => { - let layout = LayoutVerified::new_slice(bytes.as_ref()).expect("invalid layout"); - let slice = layout.into_slice(); - let setbuf = SetBuf::new_unchecked(slice.to_vec()); + let vec = match LayoutVerified::new_slice(bytes.as_ref()) { + Some(layout) => layout.into_slice().to_vec(), + None => { + let len = bytes.as_ref().len(); + let count = len / std::mem::size_of::(); + let mut buf: Vec = Vec::with_capacity(count); + unsafe { + let src = bytes.as_ref().as_ptr(); + let dst = buf.as_mut_ptr() as *mut u8; + std::ptr::copy_nonoverlapping(src, dst, len); + buf.set_len(count); + } + buf + } + }; + + let setbuf = SetBuf::new_unchecked(vec); + Ok(Some(setbuf)) }, None => Ok(None),