diff --git a/http-ui/src/main.rs b/http-ui/src/main.rs index 641f82046..57a78b41e 100644 --- a/http-ui/src/main.rs +++ b/http-ui/src/main.rs @@ -19,7 +19,7 @@ use flate2::read::GzDecoder; use futures::{stream, FutureExt, StreamExt}; use heed::EnvOpenOptions; use milli::documents::DocumentBatchReader; -use milli::tokenizer::{Analyzer, AnalyzerConfig}; +use milli::tokenizer::{Tokenizer, TokenizerBuilder}; use milli::update::UpdateIndexingStep::*; use milli::update::{ ClearDocuments, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Setting, @@ -139,17 +139,16 @@ pub struct IndexerOpt { pub max_positions_per_attributes: Option, } -struct Highlighter<'a, A> { - analyzer: Analyzer<'a, A>, +struct Highlighter<'s, A> { + tokenizer: Tokenizer<'s, A>, } -impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { - fn new(stop_words: &'a fst::Set) -> Self { - let mut config = AnalyzerConfig::default(); - config.stop_words(stop_words); - let analyzer = Analyzer::new(config); +impl<'s, A: AsRef<[u8]>> Highlighter<'s, A> { + fn new(stop_words: &'s fst::Set) -> Self { + let mut builder = TokenizerBuilder::new(); + builder.stop_words(stop_words); - Self { analyzer } + Self { tokenizer: builder.build() } } fn highlight_value(&self, value: Value, matcher_builder: &MatcherBuilder) -> Value { @@ -158,9 +157,8 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { Value::Bool(boolean) => Value::Bool(boolean), Value::Number(number) => Value::Number(number), Value::String(old_string) => { - let analyzed = self.analyzer.analyze(&old_string); - let analyzed: Vec<_> = analyzed.tokens().collect(); - let mut matcher = matcher_builder.build(&analyzed[..], &old_string); + let tokens: Vec<_> = self.tokenizer.tokenize(&old_string).collect(); + let mut matcher = matcher_builder.build(&tokens[..], &old_string); let format_options = FormatOptions { highlight: true, crop: Some(10) };