snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 02:27:40 +08:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Warn instead of returning an error when a conversion fails

Browse Source
This commit is contained in:
Kerollmops 2021-04-07 14:52:51 +02:00
parent 2aeef09316
commit c9b2d3ae1a
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
3 changed files with 33 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.lock generated
View File

@ -1521,7 +1521,8 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]] [[package]]
name = "pest" name = "pest"
version = "2.1.3" version = "2.1.3"
source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
dependencies = [ dependencies = [
"ucd-trie", "ucd-trie",
] ]
@ -1529,8 +1530,7 @@ dependencies = [
[[package]] [[package]]
name = "pest" name = "pest"
version = "2.1.3" version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67"
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
dependencies = [ dependencies = [
"ucd-trie", "ucd-trie",
] ]

2
milli/src/update/index_documents/mod.rs
View File

@ -440,6 +440,8 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
.enumerate() .enumerate()
.map(|(i, documents)| { .map(|(i, documents)| {
let store = Store::new( let store = Store::new(
primary_key.clone(),
fields_ids_map.clone(),
searchable_fields.clone(), searchable_fields.clone(),
faceted_fields.clone(), faceted_fields.clone(),
linked_hash_map_size, linked_hash_map_size,

33
milli/src/update/index_documents/store.rs
View File

@ -12,7 +12,7 @@ use fst::Set;
use grenad::{Reader, FileFuse, Writer, Sorter, CompressionType}; use grenad::{Reader, FileFuse, Writer, Sorter, CompressionType};
use heed::BytesEncode; use heed::BytesEncode;
use linked_hash_map::LinkedHashMap; use linked_hash_map::LinkedHashMap;
use log::{debug, info}; use log::{debug, info, warn};
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token, TokenKind, token::SeparatorKind}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token, TokenKind, token::SeparatorKind};
use ordered_float::OrderedFloat; use ordered_float::OrderedFloat;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
@ -24,7 +24,7 @@ use crate::heed_codec::facet::{FacetValueStringCodec, FacetLevelValueF64Codec};
use crate::heed_codec::facet::{FieldDocIdFacetStringCodec, FieldDocIdFacetF64Codec}; use crate::heed_codec::facet::{FieldDocIdFacetStringCodec, FieldDocIdFacetF64Codec};
use crate::heed_codec::{BoRoaringBitmapCodec, CboRoaringBitmapCodec}; use crate::heed_codec::{BoRoaringBitmapCodec, CboRoaringBitmapCodec};
use crate::update::UpdateIndexingStep; use crate::update::UpdateIndexingStep;
use crate::{json_to_string, SmallVec8, SmallVec32, Position, DocumentId, FieldId}; use crate::{json_to_string, SmallVec8, SmallVec32, Position, DocumentId, FieldId, FieldsIdsMap};
use super::{MergeFn, create_writer, create_sorter, writer_into_reader}; use super::{MergeFn, create_writer, create_sorter, writer_into_reader};
use super::merge_function::{ use super::merge_function::{
@ -50,6 +50,8 @@ pub struct Readers {
pub struct Store<'s, A> { pub struct Store<'s, A> {
// Indexing parameters // Indexing parameters
primary_key: String,
fields_ids_map: FieldsIdsMap,
searchable_fields: HashSet<FieldId>, searchable_fields: HashSet<FieldId>,
faceted_fields: HashMap<FieldId, FacetType>, faceted_fields: HashMap<FieldId, FacetType>,
// Caches // Caches
@ -78,6 +80,8 @@ pub struct Store<'s, A> {
impl<'s, A: AsRef<[u8]>> Store<'s, A> { impl<'s, A: AsRef<[u8]>> Store<'s, A> {
pub fn new( pub fn new(
primary_key: String,
fields_ids_map: FieldsIdsMap,
searchable_fields: HashSet<FieldId>, searchable_fields: HashSet<FieldId>,
faceted_fields: HashMap<FieldId, FacetType>, faceted_fields: HashMap<FieldId, FacetType>,
linked_hash_map_size: Option<usize>, linked_hash_map_size: Option<usize>,
@ -149,6 +153,8 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
Ok(Store { Ok(Store {
// Indexing parameters. // Indexing parameters.
primary_key,
fields_ids_map,
searchable_fields, searchable_fields,
faceted_fields, faceted_fields,
// Caches // Caches
@ -462,9 +468,26 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
let value = serde_json::from_slice(content)?; let value = serde_json::from_slice(content)?;
if let Some(ftype) = self.faceted_fields.get(&attr) { if let Some(ftype) = self.faceted_fields.get(&attr) {
let mut values = parse_facet_value(*ftype, &value).with_context(|| { let mut values = match parse_facet_value(*ftype, &value) {
format!("extracting facets from the value {}", value) Ok(values) => values,
})?; Err(e) => {
// We extract the name of the attribute and the document id
// to help users debug a facet type conversion.
let attr_name = self.fields_ids_map.name(attr).unwrap();
let document_id: Value = self.fields_ids_map.id(&self.primary_key)
.and_then(|fid| document.get(fid))
.map(serde_json::from_slice)
.unwrap()?;
let context = format!(
"while extracting facet from the {:?} attribute in the {} document",
attr_name, document_id,
);
warn!("{}", e.context(context));
SmallVec8::default()
},
};
facet_values.entry(attr).or_insert_with(SmallVec8::new).extend(values.drain(..)); facet_values.entry(attr).or_insert_with(SmallVec8::new).extend(values.drain(..));
} }