snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 03:55:07 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Use compatibility decomposition normalizer in facets

Browse Source
This commit is contained in:
ManyTheFish 2023-01-18 15:02:13 +01:00
parent e64571a881
commit d1fc42b53a
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs
View File

@ -4,6 +4,7 @@ use std::fs::File;
use std::io; use std::io;
use std::mem::size_of; use std::mem::size_of;
use charabia::normalizer::{CharNormalizer, CompatibilityDecompositionNormalizer};
use heed::zerocopy::AsBytes; use heed::zerocopy::AsBytes;
use heed::BytesEncode; use heed::BytesEncode;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
@ -135,7 +136,9 @@ fn extract_facet_values(value: &Value) -> (Vec<f64>, Vec<(String, String)>) {
} }
} }
Value::String(original) => { Value::String(original) => {
let normalized = original.trim().to_lowercase(); let normalized = CompatibilityDecompositionNormalizer
.normalize_str(original.trim())
.to_lowercase();
output_strings.push((normalized, original.clone())); output_strings.push((normalized, original.clone()));
} }
Value::Array(values) => { Value::Array(values) => {