snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 20:15:07 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge pull request #607 from tpayet/add-separators-tokenizer

Browse Source 
Add '@' char as a tokenizer separator
This commit is contained in:
Clément Renault 2020-04-16 12:18:11 +02:00 committed by GitHub
commit 10d047a636
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,3 +1,7 @@
## v0.10.1 (unreleased)
- Add '@' character as tokenizer separator (#607)
## v0.10 ## v0.10
- Refined filtering (#592) - Refined filtering (#592)

2
meilisearch-tokenizer/src/lib.rs
View File

@ -54,7 +54,7 @@ fn classify_separator(c: char) -> Option<SeparatorCategory> {
c if c.is_whitespace() => Some(Soft), // whitespaces c if c.is_whitespace() => Some(Soft), // whitespaces
c if deunicode_char(c) == Some("'") => Some(Soft), // quotes c if deunicode_char(c) == Some("'") => Some(Soft), // quotes
c if deunicode_char(c) == Some("\"") => Some(Soft), // double quotes c if deunicode_char(c) == Some("\"") => Some(Soft), // double quotes
'-' | '_' | '\'' | ':' | '/' | '\\' => Some(Soft), '-' | '_' | '\'' | ':' | '/' | '\\' | '@' => Some(Soft),
'.' | ';' | ',' | '!' | '?' | '(' | ')' => Some(Hard), '.' | ';' | ',' | '!' | '?' | '(' | ')' => Some(Hard),
_ => None, _ => None,
} }