snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 12:05:05 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Merge pull request #121 from Kerollmops/no-cjk-unidecode

Browse Source 
Do not save unidecoded cjk kanjis
This commit is contained in:
Clément Renault 2019-02-23 22:34:47 +01:00 committed by GitHub
commit 19724e5af9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/database/serde/indexer_serializer.rs
View File

@ -8,7 +8,7 @@ use crate::database::serde::SerializerError;
use crate::database::schema::SchemaAttr;
use crate::tokenizer::TokenizerBuilder;
use crate::tokenizer::Token;
use crate::{DocumentId, DocIndex};
use crate::{is_cjk, DocumentId, DocIndex};
pub struct IndexerSerializer<'a, 'b, B> {
pub tokenizer_builder: &'a B,
@ -65,13 +65,16 @@ where B: TokenizerBuilder
if self.stop_words.contains(&word_lower) { continue }
// and the unidecoded lowercased version
let word_unidecoded = unidecode::unidecode(word).to_lowercase();
if word_lower != word_unidecoded {
let char_index = char_index as u32;
let char_length = length;
if !word_lower.chars().any(is_cjk) {
let word_unidecoded = unidecode::unidecode(word).to_lowercase();
let word_unidecoded = word_unidecoded.trim();
if word_lower != word_unidecoded {
let char_index = char_index as u32;
let char_length = length;
let doc_index = DocIndex { document_id, attribute, word_index, char_index, char_length };
self.update.insert_doc_index(word_unidecoded.into_bytes(), doc_index)?;
let doc_index = DocIndex { document_id, attribute, word_index, char_index, char_length };
self.update.insert_doc_index(word_unidecoded.as_bytes().to_vec(), doc_index)?;
}
}
let char_index = char_index as u32;