snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 10:37:41 +08:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Make the tokenizer understand strange whitespaces/quotes

Browse Source
This commit is contained in:
Clément Renault 2019-11-04 16:10:13 +01:00
parent ccded7b429
commit de2b8672d4
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
meilidb-tokenizer/Cargo.toml
View File

@ -5,4 +5,5 @@ authors = ["Kerollmops <renault.cle@gmail.com>"]
edition = "2018"
[dependencies]
deunicode = "1.0.0"
slice-group-by = "0.2.4"

6
meilidb-tokenizer/src/lib.rs
View File

@ -1,4 +1,5 @@
use self::SeparatorCategory::*;
use deunicode::deunicode_char;
use slice_group_by::StrGroupBy;
use std::iter::Peekable;
@ -43,7 +44,10 @@ fn is_separator(c: char) -> bool {
fn classify_separator(c: char) -> Option<SeparatorCategory> {
match c {
' ' | '-' | '_' | '\'' | ':' | '"' => Some(Soft),
c if c.is_whitespace() => Some(Soft), // whitespaces
c if deunicode_char(c) == Some("'") => Some(Soft), // quotes
c if deunicode_char(c) == Some("\"") => Some(Soft), // double quotes
'-' | '_' | '\'' | ':' => Some(Soft),
'.' | ';' | ',' | '!' | '?' | '(' | ')' => Some(Hard),
_ => None,
}