snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 20:15:07 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

update heed

Browse Source
This commit is contained in:
mpostma 2020-12-09 11:27:38 +01:00
parent e2b0402cf5
commit 56ad400c49
10 changed files with 71 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
Cargo.lock generated
View File

@ -749,6 +749,25 @@ dependencies = [
"scopeguard",
]
[[package]]
name = "crossbeam-queue"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
dependencies = [
"crossbeam-utils 0.6.6",
]
[[package]]
name = "crossbeam-utils"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
dependencies = [
"cfg-if 0.1.10",
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
@ -968,6 +987,16 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "form_urlencoded"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00"
dependencies = [
"matches",
"percent-encoding",
]
[[package]]
name = "fs_extra"
version = "1.2.0"
@ -1188,9 +1217,9 @@ dependencies = [
[[package]]
name = "heed"
version = "0.8.1"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c9d6f92f402a1452d658038a279d882c77fc5c1f50f29eb491b5fac537f479"
checksum = "2eaba3b0edee6a9cd551f24caca2027922b03259f7203a15f0b86af4c1348fcc"
dependencies = [
"byteorder",
"heed-traits",
@ -1199,6 +1228,7 @@ dependencies = [
"lmdb-rkv-sys",
"once_cell",
"page_size",
"synchronoise",
"url",
"zerocopy",
]
@ -1211,9 +1241,9 @@ checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c"
[[package]]
name = "heed-types"
version = "0.7.0"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e100387815256b00dbb4f48db990f7fa03e9b88b4a89c2a1661b7d9d77b77c46"
checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405"
dependencies = [
"bincode",
"heed-traits",
@ -1489,9 +1519,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.79"
version = "0.2.81"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743"
checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb"
[[package]]
name = "linked-hash-map"
@ -2744,6 +2774,15 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "synchronoise"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb"
dependencies = [
"crossbeam-queue",
]
[[package]]
name = "synstructure"
version = "0.12.4"
@ -3152,10 +3191,11 @@ dependencies = [
[[package]]
name = "url"
version = "2.1.1"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb"
checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e"
dependencies = [
"form_urlencoded",
"idna",
"matches",
"percent-encoding",

2
meilisearch-core/Cargo.toml
View File

@ -18,7 +18,7 @@ either = "1.5.3"
env_logger = "0.8.2"
fst = "0.4.4"
hashbrown = { version = "0.9.0", features = ["serde"] }
heed = "0.8.0"
heed = "0.10.5"
indexmap = { version = "1.3.2", features = ["serde-1"] }
intervaltree = "0.2.5"
itertools = "0.9.0"

2
meilisearch-core/src/criterion/mod.rs
View File

@ -61,7 +61,7 @@ pub trait Criterion {
}
pub struct ContextMut<'h, 'p, 'tag, 'txn, 'q> {
pub reader: &'h heed::RoTxn<MainT>,
pub reader: &'h heed::RoTxn<'h, MainT>,
pub postings_lists: &'p mut SmallArena<'tag, PostingsListView<'txn>>,
pub query_mapping: &'q HashMap<QueryId, Range<usize>>,
pub documents_fields_counts_store: store::DocumentsFieldsCounts,

8
meilisearch-core/src/database.rs
View File

@ -22,11 +22,11 @@ type ArcSwapFn = arc_swap::ArcSwapOption<BoxUpdateFn>;
type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
pub type MainWriter<'a> = heed::RwTxn<'a, MainT>;
pub type MainReader = heed::RoTxn<MainT>;
pub type MainWriter<'a, 'b> = heed::RwTxn<'a, 'b, MainT>;
pub type MainReader<'a, 'b> = heed::RoTxn<'a, MainT>;
pub type UpdateWriter<'a> = heed::RwTxn<'a, UpdateT>;
pub type UpdateReader = heed::RoTxn<UpdateT>;
pub type UpdateWriter<'a, 'b> = heed::RwTxn<'a, 'b, UpdateT>;
pub type UpdateReader<'a> = heed::RoTxn<'a, UpdateT>;
const LAST_UPDATE_KEY: &str = "last-update";

2
meilisearch-core/src/serde/deserializer.rs
View File

@ -51,7 +51,7 @@ impl From<heed::Error> for DeserializerError {
pub struct Deserializer<'a> {
pub document_id: DocumentId,
pub reader: &'a heed::RoTxn<MainT>,
pub reader: &'a heed::RoTxn<'a, MainT>,
pub documents_fields: DocumentsFields,
pub schema: &'a Schema,
pub fields: Option<&'a HashSet<FieldId>>,

2
meilisearch-core/src/store/docs_words.rs
View File

@ -33,7 +33,7 @@ impl DocsWords {
self.docs_words.clear(writer)
}
pub fn doc_words(self, reader: &heed::RoTxn<MainT>, document_id: DocumentId) -> ZResult<FstSetCow> {
pub fn doc_words<'a>(self, reader: &'a heed::RoTxn<'a, MainT>, document_id: DocumentId) -> ZResult<FstSetCow> {
let document_id = BEU32::new(document_id.0);
match self.docs_words.get(reader, &document_id)? {
Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),

4
meilisearch-core/src/store/facets.rs
View File

@ -2,7 +2,7 @@ use std::borrow::Cow;
use std::collections::HashMap;
use std::mem;
use heed::{RwTxn, RoTxn, RoRange, types::Str, BytesEncode, BytesDecode};
use heed::{RwTxn, RoTxn, RoPrefix, types::Str, BytesEncode, BytesDecode};
use sdset::{SetBuf, Set, SetOperation};
use meilisearch_types::DocumentId;
@ -62,7 +62,7 @@ impl Facets {
Ok(self.facets.put(writer, &facet_key, &(facet_value, doc_ids))?)
}
pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, field_id: FieldId) -> MResult<RoRange<'txn, FacetKey, FacetData>> {
pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, field_id: FieldId) -> MResult<RoPrefix<'txn, FacetKey, FacetData>> {
Ok(self.facets.prefix_iter(reader, &FacetKey::new(field_id, String::new()))?)
}

10
meilisearch-core/src/store/main.rs
View File

@ -143,7 +143,7 @@ impl Main {
self.put_external_docids(writer, &external_docids)
}
pub fn external_docids(self, reader: &heed::RoTxn<MainT>) -> MResult<FstMapCow> {
pub fn external_docids<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstMapCow> {
match self.main.get::<_, Str, ByteSlice>(reader, EXTERNAL_DOCIDS_KEY)? {
Some(bytes) => Ok(fst::Map::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
None => Ok(fst::Map::default().map_data(Cow::Owned).unwrap()),
@ -155,7 +155,7 @@ impl Main {
Ok(external_ids.get(external_docid).map(|id| DocumentId(id as u32)))
}
pub fn words_fst(self, reader: &heed::RoTxn<MainT>) -> MResult<FstSetCow> {
pub fn words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstSetCow> {
match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? {
Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
@ -170,7 +170,7 @@ impl Main {
Ok(self.main.put::<_, Str, CowSlice<DocumentId>>(writer, SORTED_DOCUMENT_IDS_CACHE_KEY, documents_ids)?)
}
pub fn sorted_document_ids_cache(self, reader: &heed::RoTxn<MainT>) -> MResult<Option<Cow<[DocumentId]>>> {
pub fn sorted_document_ids_cache<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<Option<Cow<[DocumentId]>>> {
Ok(self.main.get::<_, Str, CowSlice<DocumentId>>(reader, SORTED_DOCUMENT_IDS_CACHE_KEY)?)
}
@ -199,7 +199,7 @@ impl Main {
Ok(self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)?)
}
pub(crate) fn synonyms_fst(self, reader: &heed::RoTxn<MainT>) -> MResult<FstSetCow> {
pub(crate) fn synonyms_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstSetCow> {
match self.main.get::<_, Str, ByteSlice>(reader, SYNONYMS_KEY)? {
Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),
@ -219,7 +219,7 @@ impl Main {
Ok(self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)?)
}
pub(crate) fn stop_words_fst(self, reader: &heed::RoTxn<MainT>) -> MResult<FstSetCow> {
pub(crate) fn stop_words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult<FstSetCow> {
match self.main.get::<_, Str, ByteSlice>(reader, STOP_WORDS_KEY)? {
Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()),
None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()),

12
meilisearch-core/src/update/documents_addition.rs
View File

@ -146,8 +146,8 @@ where A: AsRef<[u8]>,
Ok(())
}
pub fn apply_addition<'a, 'b>(
writer: &'a mut heed::RwTxn<'b, MainT>,
pub fn apply_addition<'a, 'b, 'c>(
writer: &'a mut heed::RwTxn<'b, 'c, MainT>,
index: &store::Index,
new_documents: Vec<IndexMap<String, Value>>,
partial: bool
@ -272,16 +272,16 @@ pub fn apply_addition<'a, 'b>(
Ok(())
}
pub fn apply_documents_partial_addition<'a, 'b>(
writer: &'a mut heed::RwTxn<'b, MainT>,
pub fn apply_documents_partial_addition<'a, 'b, 'c>(
writer: &'a mut heed::RwTxn<'b, 'c, MainT>,
index: &store::Index,
new_documents: Vec<IndexMap<String, Value>>,
) -> MResult<()> {
apply_addition(writer, index, new_documents, true)
}
pub fn apply_documents_addition<'a, 'b>(
writer: &'a mut heed::RwTxn<'b, MainT>,
pub fn apply_documents_addition<'a, 'b, 'c>(
writer: &'a mut heed::RwTxn<'b, 'c, MainT>,
index: &store::Index,
new_documents: Vec<IndexMap<String, Value>>,
) -> MResult<()> {

4
meilisearch-core/src/update/mod.rs
View File

@ -212,8 +212,8 @@ pub fn next_update_id(
Ok(new_update_id)
}
pub fn update_task<'a, 'b>(
writer: &'a mut heed::RwTxn<'b, MainT>,
pub fn update_task<'a, 'b, 'c>(
writer: &'a mut heed::RwTxn<'b, 'c, MainT>,
index: &store::Index,
update_id: u64,
update: Update,