snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 18:45:06 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge #245

Browse Source 
245: Warn for when a key is too large for LMDB r=Kerollmops a=Kerollmops

Closes #191, and resolves #140.

Co-authored-by: Kerollmops <clement@meilisearch.com>
This commit is contained in:
bors[bot] 2021-06-22 12:10:52 +00:00 committed by GitHub
commit 5b6adc6d96
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
milli/src/update/index_documents/store.rs
View File

@ -11,7 +11,7 @@ use fst::Set;
use grenad::{CompressionType, FileFuse, Reader, Sorter, Writer}; use grenad::{CompressionType, FileFuse, Reader, Sorter, Writer};
use heed::BytesEncode; use heed::BytesEncode;
use linked_hash_map::LinkedHashMap; use linked_hash_map::LinkedHashMap;
use log::{debug, info}; use log::{debug, info, warn};
use meilisearch_tokenizer::token::SeparatorKind; use meilisearch_tokenizer::token::SeparatorKind;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token, TokenKind}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token, TokenKind};
use ordered_float::OrderedFloat; use ordered_float::OrderedFloat;
@ -426,6 +426,11 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
// that we write under the generated key into MTBL // that we write under the generated key into MTBL
if lmdb_key_valid_size(&key) { if lmdb_key_valid_size(&key) {
sorter.insert(&key, &buffer)?; sorter.insert(&key, &buffer)?;
} else {
warn!(
"words pairs proximity ({:?} - {:?}, {:?}) is too large to be saved",
w1, w2, min_prox
);
} }
} }
@ -457,6 +462,8 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
// that we write under the generated key into MTBL // that we write under the generated key into MTBL
if lmdb_key_valid_size(&key) { if lmdb_key_valid_size(&key) {
writer.insert(&key, &buffer)?; writer.insert(&key, &buffer)?;
} else {
warn!("word {:?} is too large to be saved", word);
} }
} }
@ -493,6 +500,8 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
// that we write under the generated key into MTBL // that we write under the generated key into MTBL
if lmdb_key_valid_size(&key_buffer) { if lmdb_key_valid_size(&key_buffer) {
writer.insert(&key_buffer, &data_buffer)?; writer.insert(&key_buffer, &data_buffer)?;
} else {
warn!("word {:?} is too large to be saved", word);
} }
} }
} }
@ -517,6 +526,8 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
if lmdb_key_valid_size(&key_buffer) { if lmdb_key_valid_size(&key_buffer) {
sorter.insert(&key_buffer, &data_buffer)?; sorter.insert(&key_buffer, &data_buffer)?;
} else {
warn!("facet value {:?} is too large to be saved", value);
} }
} }
@ -582,6 +593,8 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
if lmdb_key_valid_size(&buffer) { if lmdb_key_valid_size(&buffer) {
sorter.insert(&buffer, &[])?; sorter.insert(&buffer, &[])?;
} else {
warn!("facet value {:?} is too large to be saved", value);
} }
Ok(()) Ok(())
@ -606,6 +619,8 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
// that we write under the generated key into MTBL // that we write under the generated key into MTBL
if lmdb_key_valid_size(&key) { if lmdb_key_valid_size(&key) {
sorter.insert(&key, &buffer)?; sorter.insert(&key, &buffer)?;
} else {
warn!("word {:?} is too large to be saved", word);
} }
} }