From 78535b3e3342d9ffaaf2a663842c02b0707ba98d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Mon, 21 Oct 2019 12:05:53 +0200 Subject: [PATCH] Move to heed 0.1.0 --- meilidb-core/Cargo.toml | 6 +-- meilidb-core/src/automaton/mod.rs | 4 +- meilidb-core/src/database.rs | 21 ++++----- meilidb-core/src/error.rs | 8 ++-- meilidb-core/src/query_builder.rs | 8 ++-- meilidb-core/src/serde/deserializer.rs | 10 ++--- meilidb-core/src/serde/mod.rs | 8 ++-- meilidb-core/src/store/docs_words.rs | 16 +++---- meilidb-core/src/store/documents_fields.rs | 16 +++---- .../src/store/documents_fields_counts.rs | 27 ++++++------ meilidb-core/src/store/main.rs | 43 ++++++++++--------- meilidb-core/src/store/mod.rs | 18 ++++---- meilidb-core/src/store/postings_lists.rs | 12 +++--- meilidb-core/src/store/synonyms.rs | 12 +++--- meilidb-core/src/store/updates.rs | 42 ++++-------------- meilidb-core/src/store/updates_results.rs | 12 +++--- meilidb-core/src/update/customs_update.rs | 6 +-- meilidb-core/src/update/documents_addition.rs | 6 +-- meilidb-core/src/update/documents_deletion.rs | 6 +-- meilidb-core/src/update/mod.rs | 11 ++--- meilidb-core/src/update/schema_update.rs | 4 +- meilidb-core/src/update/synonyms_addition.rs | 6 +-- meilidb-core/src/update/synonyms_deletion.rs | 6 +-- 23 files changed, 132 insertions(+), 176 deletions(-) diff --git a/meilidb-core/Cargo.toml b/meilidb-core/Cargo.toml index 0469996e2..2e6a5fb5d 100644 --- a/meilidb-core/Cargo.toml +++ b/meilidb-core/Cargo.toml @@ -12,6 +12,7 @@ crossbeam-channel = "0.3.9" deunicode = "1.0.0" env_logger = "0.7.0" hashbrown = { version = "0.6.0", features = ["serde"] } +heed = "0.1.0" log = "0.4.8" meilidb-schema = { path = "../meilidb-schema", version = "0.1.0" } meilidb-tokenizer = { path = "../meilidb-tokenizer", version = "0.1.0" } @@ -24,11 +25,6 @@ siphasher = "0.3.0" slice-group-by = "0.2.6" zerocopy = "0.2.8" -[dependencies.zlmdb] -package = "zerocopy-lmdb" -git = "https://github.com/Kerollmops/zerocopy-lmdb.git" -branch = "master" - [dependencies.levenshtein_automata] git = "https://github.com/Kerollmops/levenshtein-automata.git" branch = "arc-byte-slice" diff --git a/meilidb-core/src/automaton/mod.rs b/meilidb-core/src/automaton/mod.rs index da398c669..1bfb41e67 100644 --- a/meilidb-core/src/automaton/mod.rs +++ b/meilidb-core/src/automaton/mod.rs @@ -23,7 +23,7 @@ pub struct AutomatonProducer { impl AutomatonProducer { pub fn new( - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, query: &str, main_store: store::Main, synonyms_store: store::Synonyms, @@ -103,7 +103,7 @@ pub fn normalize_str(string: &str) -> String { } fn generate_automatons( - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, query: &str, main_store: store::Main, synonym_store: store::Synonyms, diff --git a/meilidb-core/src/database.rs b/meilidb-core/src/database.rs index 64da2d5f9..79f4d731e 100644 --- a/meilidb-core/src/database.rs +++ b/meilidb-core/src/database.rs @@ -5,9 +5,9 @@ use std::sync::{Arc, RwLock}; use std::{fs, thread}; use crossbeam_channel::Receiver; +use heed::types::{Str, Unit}; +use heed::{CompactionOption, Result as ZResult}; use log::{debug, error}; -use zlmdb::types::{Str, Unit}; -use zlmdb::{CompactionOption, Result as ZResult}; use crate::{store, update, Index, MResult}; @@ -15,18 +15,13 @@ pub type BoxUpdateFn = Box type ArcSwapFn = arc_swap::ArcSwapOption; pub struct Database { - pub env: zlmdb::Env, - common_store: zlmdb::DynDatabase, - indexes_store: zlmdb::Database, + pub env: heed::Env, + common_store: heed::PolyDatabase, + indexes_store: heed::Database, indexes: RwLock, thread::JoinHandle<()>)>>, } -fn update_awaiter( - receiver: Receiver<()>, - env: zlmdb::Env, - update_fn: Arc, - index: Index, -) { +fn update_awaiter(receiver: Receiver<()>, env: heed::Env, update_fn: Arc, index: Index) { for () in receiver { // consume all updates in order (oldest first) loop { @@ -67,7 +62,7 @@ impl Database { pub fn open_or_create(path: impl AsRef) -> MResult { fs::create_dir_all(path.as_ref())?; - let env = zlmdb::EnvOpenOptions::new() + let env = heed::EnvOpenOptions::new() .map_size(10 * 1024 * 1024 * 1024) // 10GB .max_dbs(3000) .open(path)?; @@ -199,7 +194,7 @@ impl Database { Ok(indexes.keys().cloned().collect()) } - pub fn common_store(&self) -> zlmdb::DynDatabase { + pub fn common_store(&self) -> heed::PolyDatabase { self.common_store } } diff --git a/meilidb-core/src/error.rs b/meilidb-core/src/error.rs index a39164341..986f1d08b 100644 --- a/meilidb-core/src/error.rs +++ b/meilidb-core/src/error.rs @@ -12,7 +12,7 @@ pub enum Error { SchemaMissing, WordIndexMissing, MissingDocumentId, - Zlmdb(zlmdb::Error), + Zlmdb(heed::Error), Fst(fst::Error), SerdeJson(SerdeJsonError), Bincode(bincode::Error), @@ -27,8 +27,8 @@ impl From for Error { } } -impl From for Error { - fn from(error: zlmdb::Error) -> Error { +impl From for Error { + fn from(error: heed::Error) -> Error { Error::Zlmdb(error) } } @@ -79,7 +79,7 @@ impl fmt::Display for Error { SchemaMissing => write!(f, "this index does not have a schema"), WordIndexMissing => write!(f, "this index does not have a word index"), MissingDocumentId => write!(f, "document id is missing"), - Zlmdb(e) => write!(f, "zlmdb error; {}", e), + Zlmdb(e) => write!(f, "heed error; {}", e), Fst(e) => write!(f, "fst error; {}", e), SerdeJson(e) => write!(f, "serde json error; {}", e), Bincode(e) => write!(f, "bincode error; {}", e), diff --git a/meilidb-core/src/query_builder.rs b/meilidb-core/src/query_builder.rs index 3d0769ec3..02c299a2a 100644 --- a/meilidb-core/src/query_builder.rs +++ b/meilidb-core/src/query_builder.rs @@ -137,7 +137,7 @@ fn multiword_rewrite_matches( } fn fetch_raw_documents( - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, automatons: &[Automaton], query_enhancer: &QueryEnhancer, searchables: Option<&ReorderedAttrs>, @@ -285,7 +285,7 @@ impl<'c, 'f, 'd> QueryBuilder<'c, 'f, 'd> { pub fn query( self, - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, query: &str, range: Range, ) -> MResult> { @@ -323,7 +323,7 @@ impl<'c, 'f, 'd> QueryBuilder<'c, 'f, 'd> { } fn raw_query<'c, FI>( - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, query: &str, range: Range, @@ -454,7 +454,7 @@ where } fn raw_query_with_distinct<'c, FI, FD>( - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, query: &str, range: Range, diff --git a/meilidb-core/src/serde/deserializer.rs b/meilidb-core/src/serde/deserializer.rs index 3c99b365b..3c6955cdf 100644 --- a/meilidb-core/src/serde/deserializer.rs +++ b/meilidb-core/src/serde/deserializer.rs @@ -14,7 +14,7 @@ use crate::DocumentId; #[derive(Debug)] pub enum DeserializerError { SerdeJson(SerdeJsonError), - Zlmdb(zlmdb::Error), + Zlmdb(heed::Error), Custom(String), } @@ -28,7 +28,7 @@ impl fmt::Display for DeserializerError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { DeserializerError::SerdeJson(e) => write!(f, "serde json related error: {}", e), - DeserializerError::Zlmdb(e) => write!(f, "zlmdb related error: {}", e), + DeserializerError::Zlmdb(e) => write!(f, "heed related error: {}", e), DeserializerError::Custom(s) => f.write_str(s), } } @@ -42,15 +42,15 @@ impl From for DeserializerError { } } -impl From for DeserializerError { - fn from(error: zlmdb::Error) -> DeserializerError { +impl From for DeserializerError { + fn from(error: heed::Error) -> DeserializerError { DeserializerError::Zlmdb(error) } } pub struct Deserializer<'a> { pub document_id: DocumentId, - pub reader: &'a zlmdb::RoTxn, + pub reader: &'a heed::RoTxn, pub documents_fields: DocumentsFields, pub schema: &'a Schema, pub attributes: Option<&'a HashSet>, diff --git a/meilidb-core/src/serde/mod.rs b/meilidb-core/src/serde/mod.rs index c2feafbf0..12ba1286a 100644 --- a/meilidb-core/src/serde/mod.rs +++ b/meilidb-core/src/serde/mod.rs @@ -35,7 +35,7 @@ use crate::{DocumentId, ParseNumberError}; pub enum SerializerError { DocumentIdNotFound, InvalidDocumentIdType, - Zlmdb(zlmdb::Error), + Zlmdb(heed::Error), SerdeJson(SerdeJsonError), ParseNumber(ParseNumberError), UnserializableType { type_name: &'static str }, @@ -59,7 +59,7 @@ impl fmt::Display for SerializerError { SerializerError::InvalidDocumentIdType => { f.write_str("document identifier can only be of type string or number") } - SerializerError::Zlmdb(e) => write!(f, "zlmdb related error: {}", e), + SerializerError::Zlmdb(e) => write!(f, "heed related error: {}", e), SerializerError::SerdeJson(e) => write!(f, "serde json error: {}", e), SerializerError::ParseNumber(e) => { write!(f, "error while trying to parse a number: {}", e) @@ -92,8 +92,8 @@ impl From for SerializerError { } } -impl From for SerializerError { - fn from(error: zlmdb::Error) -> SerializerError { +impl From for SerializerError { + fn from(error: heed::Error) -> SerializerError { SerializerError::Zlmdb(error) } } diff --git a/meilidb-core/src/store/docs_words.rs b/meilidb-core/src/store/docs_words.rs index 93d4192e3..b7ef6e542 100644 --- a/meilidb-core/src/store/docs_words.rs +++ b/meilidb-core/src/store/docs_words.rs @@ -1,18 +1,18 @@ use super::BEU64; use crate::DocumentId; +use heed::types::{ByteSlice, OwnedType}; +use heed::Result as ZResult; use std::sync::Arc; -use zlmdb::types::{ByteSlice, OwnedType}; -use zlmdb::Result as ZResult; #[derive(Copy, Clone)] pub struct DocsWords { - pub(crate) docs_words: zlmdb::Database, ByteSlice>, + pub(crate) docs_words: heed::Database, ByteSlice>, } impl DocsWords { pub fn put_doc_words( self, - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, document_id: DocumentId, words: &fst::Set, ) -> ZResult<()> { @@ -21,18 +21,14 @@ impl DocsWords { self.docs_words.put(writer, &document_id, bytes) } - pub fn del_doc_words( - self, - writer: &mut zlmdb::RwTxn, - document_id: DocumentId, - ) -> ZResult { + pub fn del_doc_words(self, writer: &mut heed::RwTxn, document_id: DocumentId) -> ZResult { let document_id = BEU64::new(document_id.0); self.docs_words.delete(writer, &document_id) } pub fn doc_words( self, - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, document_id: DocumentId, ) -> ZResult> { let document_id = BEU64::new(document_id.0); diff --git a/meilidb-core/src/store/documents_fields.rs b/meilidb-core/src/store/documents_fields.rs index 84b53351b..17fa20380 100644 --- a/meilidb-core/src/store/documents_fields.rs +++ b/meilidb-core/src/store/documents_fields.rs @@ -1,19 +1,19 @@ +use heed::types::{ByteSlice, OwnedType}; +use heed::Result as ZResult; use meilidb_schema::SchemaAttr; -use zlmdb::types::{ByteSlice, OwnedType}; -use zlmdb::Result as ZResult; use super::DocumentAttrKey; use crate::DocumentId; #[derive(Copy, Clone)] pub struct DocumentsFields { - pub(crate) documents_fields: zlmdb::Database, ByteSlice>, + pub(crate) documents_fields: heed::Database, ByteSlice>, } impl DocumentsFields { pub fn put_document_field( self, - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, document_id: DocumentId, attribute: SchemaAttr, value: &[u8], @@ -24,7 +24,7 @@ impl DocumentsFields { pub fn del_all_document_fields( self, - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, document_id: DocumentId, ) -> ZResult { let start = DocumentAttrKey::new(document_id, SchemaAttr::min()); @@ -34,7 +34,7 @@ impl DocumentsFields { pub fn document_attribute<'txn>( self, - reader: &'txn zlmdb::RoTxn, + reader: &'txn heed::RoTxn, document_id: DocumentId, attribute: SchemaAttr, ) -> ZResult> { @@ -44,7 +44,7 @@ impl DocumentsFields { pub fn document_fields<'txn>( self, - reader: &'txn zlmdb::RoTxn, + reader: &'txn heed::RoTxn, document_id: DocumentId, ) -> ZResult> { let start = DocumentAttrKey::new(document_id, SchemaAttr::min()); @@ -55,7 +55,7 @@ impl DocumentsFields { } pub struct DocumentFieldsIter<'txn> { - iter: zlmdb::RoRange<'txn, OwnedType, ByteSlice>, + iter: heed::RoRange<'txn, OwnedType, ByteSlice>, } impl<'txn> Iterator for DocumentFieldsIter<'txn> { diff --git a/meilidb-core/src/store/documents_fields_counts.rs b/meilidb-core/src/store/documents_fields_counts.rs index b765c8f25..5b7853ed6 100644 --- a/meilidb-core/src/store/documents_fields_counts.rs +++ b/meilidb-core/src/store/documents_fields_counts.rs @@ -1,18 +1,18 @@ use super::DocumentAttrKey; use crate::DocumentId; +use heed::types::OwnedType; +use heed::Result as ZResult; use meilidb_schema::SchemaAttr; -use zlmdb::types::OwnedType; -use zlmdb::Result as ZResult; #[derive(Copy, Clone)] pub struct DocumentsFieldsCounts { - pub(crate) documents_fields_counts: zlmdb::Database, OwnedType>, + pub(crate) documents_fields_counts: heed::Database, OwnedType>, } impl DocumentsFieldsCounts { pub fn put_document_field_count( self, - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, document_id: DocumentId, attribute: SchemaAttr, value: u64, @@ -23,7 +23,7 @@ impl DocumentsFieldsCounts { pub fn del_all_document_fields_counts( self, - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, document_id: DocumentId, ) -> ZResult { let start = DocumentAttrKey::new(document_id, SchemaAttr::min()); @@ -34,7 +34,7 @@ impl DocumentsFieldsCounts { pub fn document_field_count( self, - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, document_id: DocumentId, attribute: SchemaAttr, ) -> ZResult> { @@ -47,7 +47,7 @@ impl DocumentsFieldsCounts { pub fn document_fields_counts<'txn>( self, - reader: &'txn zlmdb::RoTxn, + reader: &'txn heed::RoTxn, document_id: DocumentId, ) -> ZResult> { let start = DocumentAttrKey::new(document_id, SchemaAttr::min()); @@ -56,10 +56,7 @@ impl DocumentsFieldsCounts { Ok(DocumentFieldsCountsIter { iter }) } - pub fn documents_ids<'txn>( - self, - reader: &'txn zlmdb::RoTxn, - ) -> ZResult> { + pub fn documents_ids<'txn>(self, reader: &'txn heed::RoTxn) -> ZResult> { let iter = self.documents_fields_counts.iter(reader)?; Ok(DocumentsIdsIter { last_seen_id: None, @@ -69,7 +66,7 @@ impl DocumentsFieldsCounts { pub fn all_documents_fields_counts<'txn>( self, - reader: &'txn zlmdb::RoTxn, + reader: &'txn heed::RoTxn, ) -> ZResult> { let iter = self.documents_fields_counts.iter(reader)?; Ok(AllDocumentsFieldsCountsIter { iter }) @@ -77,7 +74,7 @@ impl DocumentsFieldsCounts { } pub struct DocumentFieldsCountsIter<'txn> { - iter: zlmdb::RoRange<'txn, OwnedType, OwnedType>, + iter: heed::RoRange<'txn, OwnedType, OwnedType>, } impl Iterator for DocumentFieldsCountsIter<'_> { @@ -97,7 +94,7 @@ impl Iterator for DocumentFieldsCountsIter<'_> { pub struct DocumentsIdsIter<'txn> { last_seen_id: Option, - iter: zlmdb::RoIter<'txn, OwnedType, OwnedType>, + iter: heed::RoIter<'txn, OwnedType, OwnedType>, } impl Iterator for DocumentsIdsIter<'_> { @@ -121,7 +118,7 @@ impl Iterator for DocumentsIdsIter<'_> { } pub struct AllDocumentsFieldsCountsIter<'txn> { - iter: zlmdb::RoIter<'txn, OwnedType, OwnedType>, + iter: heed::RoIter<'txn, OwnedType, OwnedType>, } impl<'r> Iterator for AllDocumentsFieldsCountsIter<'r> { diff --git a/meilidb-core/src/store/main.rs b/meilidb-core/src/store/main.rs index 6e99ac067..416fcfe37 100644 --- a/meilidb-core/src/store/main.rs +++ b/meilidb-core/src/store/main.rs @@ -1,8 +1,8 @@ use crate::RankedMap; +use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str}; +use heed::Result as ZResult; use meilidb_schema::Schema; use std::sync::Arc; -use zlmdb::types::{ByteSlice, OwnedType, Serde, Str}; -use zlmdb::Result as ZResult; const CUSTOMS_KEY: &str = "customs-key"; const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents"; @@ -13,16 +13,16 @@ const WORDS_KEY: &str = "words"; #[derive(Copy, Clone)] pub struct Main { - pub(crate) main: zlmdb::DynDatabase, + pub(crate) main: heed::PolyDatabase, } impl Main { - pub fn put_words_fst(self, writer: &mut zlmdb::RwTxn, fst: &fst::Set) -> ZResult<()> { + pub fn put_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> { let bytes = fst.as_fst().as_bytes(); self.main.put::(writer, WORDS_KEY, bytes) } - pub fn words_fst(self, reader: &zlmdb::RoTxn) -> ZResult> { + pub fn words_fst(self, reader: &heed::RoTxn) -> ZResult> { match self.main.get::(reader, WORDS_KEY)? { Some(bytes) => { let len = bytes.len(); @@ -34,31 +34,32 @@ impl Main { } } - pub fn put_schema(self, writer: &mut zlmdb::RwTxn, schema: &Schema) -> ZResult<()> { + pub fn put_schema(self, writer: &mut heed::RwTxn, schema: &Schema) -> ZResult<()> { self.main - .put::>(writer, SCHEMA_KEY, schema) + .put::>(writer, SCHEMA_KEY, schema) } - pub fn schema(self, reader: &zlmdb::RoTxn) -> ZResult> { - self.main.get::>(reader, SCHEMA_KEY) - } - - pub fn put_ranked_map(self, writer: &mut zlmdb::RwTxn, ranked_map: &RankedMap) -> ZResult<()> { + pub fn schema(self, reader: &heed::RoTxn) -> ZResult> { self.main - .put::>(writer, RANKED_MAP_KEY, &ranked_map) + .get::>(reader, SCHEMA_KEY) } - pub fn ranked_map(self, reader: &zlmdb::RoTxn) -> ZResult> { + pub fn put_ranked_map(self, writer: &mut heed::RwTxn, ranked_map: &RankedMap) -> ZResult<()> { self.main - .get::>(reader, RANKED_MAP_KEY) + .put::>(writer, RANKED_MAP_KEY, &ranked_map) } - pub fn put_synonyms_fst(self, writer: &mut zlmdb::RwTxn, fst: &fst::Set) -> ZResult<()> { + pub fn ranked_map(self, reader: &heed::RoTxn) -> ZResult> { + self.main + .get::>(reader, RANKED_MAP_KEY) + } + + pub fn put_synonyms_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> { let bytes = fst.as_fst().as_bytes(); self.main.put::(writer, SYNONYMS_KEY, bytes) } - pub fn synonyms_fst(self, reader: &zlmdb::RoTxn) -> ZResult> { + pub fn synonyms_fst(self, reader: &heed::RoTxn) -> ZResult> { match self.main.get::(reader, SYNONYMS_KEY)? { Some(bytes) => { let len = bytes.len(); @@ -70,7 +71,7 @@ impl Main { } } - pub fn put_number_of_documents(self, writer: &mut zlmdb::RwTxn, f: F) -> ZResult + pub fn put_number_of_documents(self, writer: &mut heed::RwTxn, f: F) -> ZResult where F: Fn(u64) -> u64, { @@ -80,7 +81,7 @@ impl Main { Ok(new) } - pub fn number_of_documents(self, reader: &zlmdb::RoTxn) -> ZResult { + pub fn number_of_documents(self, reader: &heed::RoTxn) -> ZResult { match self .main .get::>(reader, NUMBER_OF_DOCUMENTS_KEY)? @@ -90,12 +91,12 @@ impl Main { } } - pub fn put_customs(self, writer: &mut zlmdb::RwTxn, customs: &[u8]) -> ZResult<()> { + pub fn put_customs(self, writer: &mut heed::RwTxn, customs: &[u8]) -> ZResult<()> { self.main .put::(writer, CUSTOMS_KEY, customs) } - pub fn customs<'txn>(self, reader: &'txn zlmdb::RoTxn) -> ZResult> { + pub fn customs<'txn>(self, reader: &'txn heed::RoTxn) -> ZResult> { self.main.get::(reader, CUSTOMS_KEY) } } diff --git a/meilidb-core/src/store/mod.rs b/meilidb-core/src/store/mod.rs index 68170e64e..ad14d46c5 100644 --- a/meilidb-core/src/store/mod.rs +++ b/meilidb-core/src/store/mod.rs @@ -20,10 +20,10 @@ pub use self::updates_results::UpdatesResults; use std::collections::HashSet; +use heed::Result as ZResult; use meilidb_schema::{Schema, SchemaAttr}; use serde::de; use zerocopy::{AsBytes, FromBytes}; -use zlmdb::Result as ZResult; use crate::criterion::Criteria; use crate::serde::Deserializer; @@ -97,7 +97,7 @@ pub struct Index { impl Index { pub fn document( &self, - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, attributes: Option<&HashSet<&str>>, document_id: DocumentId, ) -> MResult> { @@ -127,7 +127,7 @@ impl Index { pub fn document_attribute( &self, - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, document_id: DocumentId, attribute: SchemaAttr, ) -> MResult> { @@ -140,12 +140,12 @@ impl Index { } } - pub fn schema_update(&self, writer: &mut zlmdb::RwTxn, schema: Schema) -> MResult { + pub fn schema_update(&self, writer: &mut heed::RwTxn, schema: Schema) -> MResult { let _ = self.updates_notifier.send(()); update::push_schema_update(writer, self.updates, self.updates_results, schema) } - pub fn customs_update(&self, writer: &mut zlmdb::RwTxn, customs: Vec) -> ZResult { + pub fn customs_update(&self, writer: &mut heed::RwTxn, customs: Vec) -> ZResult { let _ = self.updates_notifier.send(()); update::push_customs_update(writer, self.updates, self.updates_results, customs) } @@ -182,7 +182,7 @@ impl Index { ) } - pub fn current_update_id(&self, reader: &zlmdb::RoTxn) -> MResult> { + pub fn current_update_id(&self, reader: &heed::RoTxn) -> MResult> { match self.updates.last_update_id(reader)? { Some((id, _)) => Ok(Some(id)), None => Ok(None), @@ -191,7 +191,7 @@ impl Index { pub fn update_status( &self, - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, update_id: u64, ) -> MResult { update::update_status(reader, self.updates, self.updates_results, update_id) @@ -221,7 +221,7 @@ impl Index { } pub fn create( - env: &zlmdb::Env, + env: &heed::Env, name: &str, updates_notifier: crossbeam_channel::Sender<()>, ) -> MResult { @@ -261,7 +261,7 @@ pub fn create( } pub fn open( - env: &zlmdb::Env, + env: &heed::Env, name: &str, updates_notifier: crossbeam_channel::Sender<()>, ) -> MResult> { diff --git a/meilidb-core/src/store/postings_lists.rs b/meilidb-core/src/store/postings_lists.rs index 8835a504a..8f8bc919e 100644 --- a/meilidb-core/src/store/postings_lists.rs +++ b/meilidb-core/src/store/postings_lists.rs @@ -1,31 +1,31 @@ use crate::DocIndex; +use heed::types::{ByteSlice, CowSlice}; +use heed::Result as ZResult; use sdset::{Set, SetBuf}; use std::borrow::Cow; -use zlmdb::types::{ByteSlice, CowSlice}; -use zlmdb::Result as ZResult; #[derive(Copy, Clone)] pub struct PostingsLists { - pub(crate) postings_lists: zlmdb::Database>, + pub(crate) postings_lists: heed::Database>, } impl PostingsLists { pub fn put_postings_list( self, - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, word: &[u8], words_indexes: &Set, ) -> ZResult<()> { self.postings_lists.put(writer, word, words_indexes) } - pub fn del_postings_list(self, writer: &mut zlmdb::RwTxn, word: &[u8]) -> ZResult { + pub fn del_postings_list(self, writer: &mut heed::RwTxn, word: &[u8]) -> ZResult { self.postings_lists.delete(writer, word) } pub fn postings_list<'txn>( self, - reader: &'txn zlmdb::RoTxn, + reader: &'txn heed::RoTxn, word: &[u8], ) -> ZResult>>> { match self.postings_lists.get(reader, word)? { diff --git a/meilidb-core/src/store/synonyms.rs b/meilidb-core/src/store/synonyms.rs index b8002d464..80b8bba67 100644 --- a/meilidb-core/src/store/synonyms.rs +++ b/meilidb-core/src/store/synonyms.rs @@ -1,16 +1,16 @@ +use heed::types::ByteSlice; +use heed::Result as ZResult; use std::sync::Arc; -use zlmdb::types::ByteSlice; -use zlmdb::Result as ZResult; #[derive(Copy, Clone)] pub struct Synonyms { - pub(crate) synonyms: zlmdb::Database, + pub(crate) synonyms: heed::Database, } impl Synonyms { pub fn put_synonyms( self, - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, word: &[u8], synonyms: &fst::Set, ) -> ZResult<()> { @@ -18,11 +18,11 @@ impl Synonyms { self.synonyms.put(writer, word, bytes) } - pub fn del_synonyms(self, writer: &mut zlmdb::RwTxn, word: &[u8]) -> ZResult { + pub fn del_synonyms(self, writer: &mut heed::RwTxn, word: &[u8]) -> ZResult { self.synonyms.delete(writer, word) } - pub fn synonyms(self, reader: &zlmdb::RoTxn, word: &[u8]) -> ZResult> { + pub fn synonyms(self, reader: &heed::RoTxn, word: &[u8]) -> ZResult> { match self.synonyms.get(reader, word)? { Some(bytes) => { let len = bytes.len(); diff --git a/meilidb-core/src/store/updates.rs b/meilidb-core/src/store/updates.rs index 8afe95e0d..e4e634dcd 100644 --- a/meilidb-core/src/store/updates.rs +++ b/meilidb-core/src/store/updates.rs @@ -1,42 +1,16 @@ use super::BEU64; use crate::update::Update; -use serde::{Deserialize, Serialize}; -use std::borrow::Cow; -use zlmdb::types::OwnedType; -use zlmdb::{BytesDecode, BytesEncode, Result as ZResult}; - -pub struct SerdeJson(std::marker::PhantomData); - -impl BytesEncode for SerdeJson -where - T: Serialize, -{ - type EItem = T; - - fn bytes_encode(item: &Self::EItem) -> Option> { - serde_json::to_vec(item).map(Cow::Owned).ok() - } -} - -impl<'a, T: 'a> BytesDecode<'a> for SerdeJson -where - T: Deserialize<'a> + Clone, -{ - type DItem = T; - - fn bytes_decode(bytes: &'a [u8]) -> Option { - serde_json::from_slice(bytes).ok() - } -} +use heed::types::{OwnedType, SerdeJson}; +use heed::Result as ZResult; #[derive(Copy, Clone)] pub struct Updates { - pub(crate) updates: zlmdb::Database, SerdeJson>, + pub(crate) updates: heed::Database, SerdeJson>, } impl Updates { // TODO do not trigger deserialize if possible - pub fn last_update_id(self, reader: &zlmdb::RoTxn) -> ZResult> { + pub fn last_update_id(self, reader: &heed::RoTxn) -> ZResult> { match self.updates.last(reader)? { Some((key, data)) => Ok(Some((key.get(), data))), None => Ok(None), @@ -44,7 +18,7 @@ impl Updates { } // TODO do not trigger deserialize if possible - fn first_update_id(self, reader: &zlmdb::RoTxn) -> ZResult> { + fn first_update_id(self, reader: &heed::RoTxn) -> ZResult> { match self.updates.first(reader)? { Some((key, data)) => Ok(Some((key.get(), data))), None => Ok(None), @@ -52,14 +26,14 @@ impl Updates { } // TODO do not trigger deserialize if possible - pub fn contains(self, reader: &zlmdb::RoTxn, update_id: u64) -> ZResult { + pub fn contains(self, reader: &heed::RoTxn, update_id: u64) -> ZResult { let update_id = BEU64::new(update_id); self.updates.get(reader, &update_id).map(|v| v.is_some()) } pub fn put_update( self, - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, update_id: u64, update: &Update, ) -> ZResult<()> { @@ -68,7 +42,7 @@ impl Updates { self.updates.put(writer, &update_id, update) } - pub fn pop_front(self, writer: &mut zlmdb::RwTxn) -> ZResult> { + pub fn pop_front(self, writer: &mut heed::RwTxn) -> ZResult> { match self.first_update_id(writer)? { Some((update_id, update)) => { let key = BEU64::new(update_id); diff --git a/meilidb-core/src/store/updates_results.rs b/meilidb-core/src/store/updates_results.rs index cd3c96075..51db65cc1 100644 --- a/meilidb-core/src/store/updates_results.rs +++ b/meilidb-core/src/store/updates_results.rs @@ -1,15 +1,15 @@ use super::BEU64; use crate::update::UpdateResult; -use zlmdb::types::{OwnedType, Serde}; -use zlmdb::Result as ZResult; +use heed::types::{OwnedType, SerdeBincode}; +use heed::Result as ZResult; #[derive(Copy, Clone)] pub struct UpdatesResults { - pub(crate) updates_results: zlmdb::Database, Serde>, + pub(crate) updates_results: heed::Database, SerdeBincode>, } impl UpdatesResults { - pub fn last_update_id(self, reader: &zlmdb::RoTxn) -> ZResult> { + pub fn last_update_id(self, reader: &heed::RoTxn) -> ZResult> { match self.updates_results.last(reader)? { Some((key, data)) => Ok(Some((key.get(), data))), None => Ok(None), @@ -18,7 +18,7 @@ impl UpdatesResults { pub fn put_update_result( self, - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, update_id: u64, update_result: &UpdateResult, ) -> ZResult<()> { @@ -28,7 +28,7 @@ impl UpdatesResults { pub fn update_result( self, - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, update_id: u64, ) -> ZResult> { let update_id = BEU64::new(update_id); diff --git a/meilidb-core/src/update/customs_update.rs b/meilidb-core/src/update/customs_update.rs index 5072dc096..72ecfba71 100644 --- a/meilidb-core/src/update/customs_update.rs +++ b/meilidb-core/src/update/customs_update.rs @@ -1,9 +1,9 @@ use crate::store; use crate::update::{next_update_id, Update}; -use zlmdb::Result as ZResult; +use heed::Result as ZResult; pub fn apply_customs_update( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, main_store: store::Main, customs: &[u8], ) -> ZResult<()> { @@ -11,7 +11,7 @@ pub fn apply_customs_update( } pub fn push_customs_update( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, customs: Vec, diff --git a/meilidb-core/src/update/documents_addition.rs b/meilidb-core/src/update/documents_addition.rs index 22769fbe2..656dcba5b 100644 --- a/meilidb-core/src/update/documents_addition.rs +++ b/meilidb-core/src/update/documents_addition.rs @@ -35,7 +35,7 @@ impl DocumentsAddition { self.documents.push(document); } - pub fn finalize(self, writer: &mut zlmdb::RwTxn) -> MResult + pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult where D: serde::Serialize, { @@ -57,7 +57,7 @@ impl Extend for DocumentsAddition { } pub fn push_documents_addition( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, addition: Vec, @@ -78,7 +78,7 @@ pub fn push_documents_addition( } pub fn apply_documents_addition( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, main_store: store::Main, documents_fields_store: store::DocumentsFields, documents_fields_counts_store: store::DocumentsFieldsCounts, diff --git a/meilidb-core/src/update/documents_deletion.rs b/meilidb-core/src/update/documents_deletion.rs index e640cb508..e1ec71ddc 100644 --- a/meilidb-core/src/update/documents_deletion.rs +++ b/meilidb-core/src/update/documents_deletion.rs @@ -49,7 +49,7 @@ impl DocumentsDeletion { Ok(()) } - pub fn finalize(self, writer: &mut zlmdb::RwTxn) -> MResult { + pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult { let _ = self.updates_notifier.send(()); let update_id = push_documents_deletion( writer, @@ -68,7 +68,7 @@ impl Extend for DocumentsDeletion { } pub fn push_documents_deletion( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, deletion: Vec, @@ -82,7 +82,7 @@ pub fn push_documents_deletion( } pub fn apply_documents_deletion( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, main_store: store::Main, documents_fields_store: store::DocumentsFields, documents_fields_counts_store: store::DocumentsFieldsCounts, diff --git a/meilidb-core/src/update/mod.rs b/meilidb-core/src/update/mod.rs index ea93413f1..8958074ec 100644 --- a/meilidb-core/src/update/mod.rs +++ b/meilidb-core/src/update/mod.rs @@ -16,9 +16,9 @@ use std::cmp; use std::collections::BTreeMap; use std::time::{Duration, Instant}; +use heed::Result as ZResult; use log::debug; use serde::{Deserialize, Serialize}; -use zlmdb::Result as ZResult; use crate::{store, DocumentId, MResult, RankedMap}; use meilidb_schema::Schema; @@ -64,7 +64,7 @@ pub enum UpdateStatus { } pub fn update_status( - reader: &zlmdb::RoTxn, + reader: &heed::RoTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, update_id: u64, @@ -82,7 +82,7 @@ pub fn update_status( } pub fn next_update_id( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, ) -> ZResult { @@ -98,10 +98,7 @@ pub fn next_update_id( Ok(new_update_id) } -pub fn update_task( - writer: &mut zlmdb::RwTxn, - index: store::Index, -) -> MResult> { +pub fn update_task(writer: &mut heed::RwTxn, index: store::Index) -> MResult> { let (update_id, update) = match index.updates.pop_front(writer)? { Some(value) => value, None => return Ok(None), diff --git a/meilidb-core/src/update/schema_update.rs b/meilidb-core/src/update/schema_update.rs index 652ca06dd..b36370eda 100644 --- a/meilidb-core/src/update/schema_update.rs +++ b/meilidb-core/src/update/schema_update.rs @@ -3,7 +3,7 @@ use crate::{error::UnsupportedOperation, store, MResult}; use meilidb_schema::Schema; pub fn apply_schema_update( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, main_store: store::Main, new_schema: &Schema, ) -> MResult<()> { @@ -17,7 +17,7 @@ pub fn apply_schema_update( } pub fn push_schema_update( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, schema: Schema, diff --git a/meilidb-core/src/update/synonyms_addition.rs b/meilidb-core/src/update/synonyms_addition.rs index f32475a1e..656e5e727 100644 --- a/meilidb-core/src/update/synonyms_addition.rs +++ b/meilidb-core/src/update/synonyms_addition.rs @@ -42,7 +42,7 @@ impl SynonymsAddition { .extend(alternatives); } - pub fn finalize(self, writer: &mut zlmdb::RwTxn) -> MResult { + pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult { let _ = self.updates_notifier.send(()); let update_id = push_synonyms_addition( writer, @@ -55,7 +55,7 @@ impl SynonymsAddition { } pub fn push_synonyms_addition( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, addition: BTreeMap>, @@ -69,7 +69,7 @@ pub fn push_synonyms_addition( } pub fn apply_synonyms_addition( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, main_store: store::Main, synonyms_store: store::Synonyms, addition: BTreeMap>, diff --git a/meilidb-core/src/update/synonyms_deletion.rs b/meilidb-core/src/update/synonyms_deletion.rs index c498c3ab0..b76f3f761 100644 --- a/meilidb-core/src/update/synonyms_deletion.rs +++ b/meilidb-core/src/update/synonyms_deletion.rs @@ -49,7 +49,7 @@ impl SynonymsDeletion { } } - pub fn finalize(self, writer: &mut zlmdb::RwTxn) -> MResult { + pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult { let _ = self.updates_notifier.send(()); let update_id = push_synonyms_deletion( writer, @@ -62,7 +62,7 @@ impl SynonymsDeletion { } pub fn push_synonyms_deletion( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, deletion: BTreeMap>>, @@ -76,7 +76,7 @@ pub fn push_synonyms_deletion( } pub fn apply_synonyms_deletion( - writer: &mut zlmdb::RwTxn, + writer: &mut heed::RwTxn, main_store: store::Main, synonyms_store: store::Synonyms, deletion: BTreeMap>>,