snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-27 04:25:06 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Update http-ui

Browse Source
This commit is contained in:
ManyTheFish 2022-06-02 18:15:36 +02:00
parent 727d663f28
commit a5c790bf4b
1 changed files with 15 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
http-ui/src/main.rs
View File

@ -19,7 +19,7 @@ use flate2::read::GzDecoder;
use futures::{stream, FutureExt, StreamExt}; use futures::{stream, FutureExt, StreamExt};
use heed::EnvOpenOptions; use heed::EnvOpenOptions;
use milli::documents::DocumentBatchReader; use milli::documents::DocumentBatchReader;
use milli::tokenizer::{Tokenizer, TokenizerBuilder}; use milli::tokenizer::TokenizerBuilder;
use milli::update::UpdateIndexingStep::*; use milli::update::UpdateIndexingStep::*;
use milli::update::{ use milli::update::{
ClearDocuments, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Setting, ClearDocuments, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Setting,
@ -140,38 +140,31 @@ pub struct IndexerOpt {
} }
struct Highlighter<'s, A> { struct Highlighter<'s, A> {
tokenizer: Tokenizer<'s, A>, matcher_builder: MatcherBuilder<'s, A>,
} }
impl<'s, A: AsRef<[u8]>> Highlighter<'s, A> { impl<'s, A: AsRef<[u8]>> Highlighter<'s, A> {
fn new(stop_words: &'s fst::Set<A>) -> Self { fn new(matcher_builder: MatcherBuilder<'s, A>) -> Self {
let mut builder = TokenizerBuilder::new(); Self { matcher_builder }
builder.stop_words(stop_words);
Self { tokenizer: builder.build() }
} }
fn highlight_value(&self, value: Value, matcher_builder: &MatcherBuilder) -> Value { fn highlight_value(&self, value: Value) -> Value {
match value { match value {
Value::Null => Value::Null, Value::Null => Value::Null,
Value::Bool(boolean) => Value::Bool(boolean), Value::Bool(boolean) => Value::Bool(boolean),
Value::Number(number) => Value::Number(number), Value::Number(number) => Value::Number(number),
Value::String(old_string) => { Value::String(old_string) => {
let tokens: Vec<_> = self.tokenizer.tokenize(&old_string).collect(); let mut matcher = self.matcher_builder.build(&old_string);
let mut matcher = matcher_builder.build(&tokens[..], &old_string);
let format_options = FormatOptions { highlight: true, crop: Some(10) }; let format_options = FormatOptions { highlight: true, crop: Some(10) };
Value::String(matcher.format(format_options).to_string()) Value::String(matcher.format(format_options).to_string())
} }
Value::Array(values) => Value::Array( Value::Array(values) => {
values.into_iter().map(|v| self.highlight_value(v, matcher_builder)).collect(), Value::Array(values.into_iter().map(|v| self.highlight_value(v)).collect())
), }
Value::Object(object) => Value::Object( Value::Object(object) => Value::Object(
object object.into_iter().map(|(k, v)| (k, self.highlight_value(v))).collect(),
.into_iter()
.map(|(k, v)| (k, self.highlight_value(v, matcher_builder)))
.collect(),
), ),
} }
} }
@ -179,14 +172,13 @@ impl<'s, A: AsRef<[u8]>> Highlighter<'s, A> {
fn highlight_record( fn highlight_record(
&self, &self,
object: &mut Map<String, Value>, object: &mut Map<String, Value>,
matcher_builder: &MatcherBuilder,
attributes_to_highlight: &HashSet<String>, attributes_to_highlight: &HashSet<String>,
) { ) {
// TODO do we need to create a string for element that are not and needs to be highlight? // TODO do we need to create a string for element that are not and needs to be highlight?
for (key, value) in object.iter_mut() { for (key, value) in object.iter_mut() {
if attributes_to_highlight.contains(key) { if attributes_to_highlight.contains(key) {
let old_value = mem::take(value); let old_value = mem::take(value);
*value = self.highlight_value(old_value, matcher_builder); *value = self.highlight_value(old_value);
} }
} }
} }
@ -798,20 +790,15 @@ async fn main() -> anyhow::Result<()> {
None => fields_ids_map.iter().map(|(_, name)| name).map(String::from).collect(), None => fields_ids_map.iter().map(|(_, name)| name).map(String::from).collect(),
}; };
let stop_words = fst::Set::default(); let mut matcher_builder =
let highlighter = Highlighter::new(&stop_words); MatcherBuilder::new(matching_words, TokenizerBuilder::default().build());
let mut matcher_builder = MatcherBuilder::from_matching_words(matching_words);
matcher_builder.highlight_prefix("<mark>".to_string()); matcher_builder.highlight_prefix("<mark>".to_string());
matcher_builder.highlight_suffix("</mark>".to_string()); matcher_builder.highlight_suffix("</mark>".to_string());
let highlighter = Highlighter::new(matcher_builder);
for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() {
let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap();
if !disable_highlighting { if !disable_highlighting {
highlighter.highlight_record( highlighter.highlight_record(&mut object, &attributes_to_highlight);
&mut object,
&matcher_builder,
&attributes_to_highlight,
);
} }
documents.push(object); documents.push(object);