snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 18:17:39 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Update milli/src/update/index_documents/extract/extract_docid_word_positions.rs

Browse Source 
Co-authored-by: Tamo <tamo@meilisearch.com>
This commit is contained in:
Many the fish 2023-08-10 10:05:03 +02:00 committed by GitHub
parent 4a21fecf67
commit 43c13faeda
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 0 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
milli/src/update/index_documents/extract/extract_docid_word_positions.rs
View File

@ -55,11 +55,9 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
tokenizer_builder.stop_words(stop_words);
}
if let Some(dictionary) = dictionary {
// let dictionary: Vec<_> = dictionary.iter().map(String::as_str).collect();
tokenizer_builder.words_dict(dictionary.as_slice());
}
if let Some(separators) = allowed_separators {
// let separators: Vec<_> = separators.iter().map(String::as_str).collect();
tokenizer_builder.separators(separators.as_slice());
}
let tokenizer = tokenizer_builder.build();