diff --git a/milli/src/search/matches/mod.rs b/milli/src/search/matches/mod.rs
index 85e77e15b..1a6d8958a 100644
--- a/milli/src/search/matches/mod.rs
+++ b/milli/src/search/matches/mod.rs
@@ -1,6 +1,6 @@
use std::borrow::Cow;
-use charabia::{SeparatorKind, Token};
+use charabia::{SeparatorKind, Token, Tokenizer};
use matching_words::{MatchType, PartialMatch, PrimitiveWordId};
pub use matching_words::{MatchingWord, MatchingWords};
use serde::Serialize;
@@ -11,16 +11,23 @@ const DEFAULT_CROP_MARKER: &'static str = "…";
const DEFAULT_HIGHLIGHT_PREFIX: &'static str = "";
const DEFAULT_HIGHLIGHT_SUFFIX: &'static str = "";
-pub struct MatcherBuilder {
+pub struct MatcherBuilder<'a, A> {
matching_words: MatchingWords,
+ tokenizer: Tokenizer<'a, A>,
crop_marker: Option,
highlight_prefix: Option,
highlight_suffix: Option,
}
-impl MatcherBuilder {
- pub fn from_matching_words(matching_words: MatchingWords) -> Self {
- Self { matching_words, crop_marker: None, highlight_prefix: None, highlight_suffix: None }
+impl<'a, A> MatcherBuilder<'a, A> {
+ pub fn new(matching_words: MatchingWords, tokenizer: Tokenizer<'a, A>) -> Self {
+ Self {
+ matching_words,
+ tokenizer,
+ crop_marker: None,
+ highlight_prefix: None,
+ highlight_suffix: None,
+ }
}
pub fn crop_marker(&mut self, marker: String) -> &Self {
@@ -38,7 +45,7 @@ impl MatcherBuilder {
self
}
- pub fn build<'t, 'm>(&'m self, tokens: &'t [Token], text: &'t str) -> Matcher<'t, 'm> {
+ pub fn build<'t, 'm>(&'m self, text: &'t str) -> Matcher<'t, 'm, A> {
let crop_marker = match &self.crop_marker {
Some(marker) => marker.as_str(),
None => &DEFAULT_CROP_MARKER,
@@ -54,8 +61,8 @@ impl MatcherBuilder {
};
Matcher {
text,
- tokens,
matching_words: &self.matching_words,
+ tokenizer: &self.tokenizer,
crop_marker,
highlight_prefix,
highlight_suffix,
@@ -93,17 +100,17 @@ pub struct MatchBounds {
pub length: usize,
}
-pub struct Matcher<'t, 'm> {
+pub struct Matcher<'t, 'm, A> {
text: &'t str,
- tokens: &'t [Token<'t>],
matching_words: &'m MatchingWords,
+ tokenizer: &'m Tokenizer<'m, A>,
crop_marker: &'m str,
highlight_prefix: &'m str,
highlight_suffix: &'m str,
- matches: Option>,
+ matches: Option<(Vec>, Vec)>,
}
-impl<'t> Matcher<'t, '_> {
+impl<'t, A: AsRef<[u8]>> Matcher<'t, '_, A> {
/// Iterates over tokens and save any of them that matches the query.
fn compute_matches(&mut self) -> &mut Self {
fn compute_partial_match<'a>(
@@ -159,10 +166,10 @@ impl<'t> Matcher<'t, '_> {
false
}
+ let tokens: Vec<_> = self.tokenizer.tokenize(self.text).collect();
let mut matches = Vec::new();
- let mut words_positions = self
- .tokens
+ let mut words_positions = tokens
.iter()
.scan((0, 0), |(token_position, word_position), token| {
let current_token_position = *token_position;
@@ -210,7 +217,7 @@ impl<'t> Matcher<'t, '_> {
}
}
- self.matches = Some(matches);
+ self.matches = Some((tokens, matches));
self
}
@@ -218,10 +225,10 @@ impl<'t> Matcher<'t, '_> {
pub fn matches(&mut self) -> Vec {
match &self.matches {
None => self.compute_matches().matches(),
- Some(matches) => matches
+ Some((tokens, matches)) => matches
.iter()
.map(|m| MatchBounds {
- start: self.tokens[m.token_position].byte_start,
+ start: tokens[m.token_position].byte_start,
length: m.match_len,
})
.collect(),
@@ -229,7 +236,7 @@ impl<'t> Matcher<'t, '_> {
}
/// Returns the bounds in byte index of the crop window.
- fn crop_bounds(&self, matches: &[Match], crop_size: usize) -> (usize, usize) {
+ fn crop_bounds(&self, tokens: &[Token], matches: &[Match], crop_size: usize) -> (usize, usize) {
// if there is no match, we start from the beginning of the string by default.
let first_match_word_position = matches.first().map(|m| m.word_position).unwrap_or(0);
let first_match_token_position = matches.first().map(|m| m.token_position).unwrap_or(0);
@@ -239,8 +246,8 @@ impl<'t> Matcher<'t, '_> {
// matches needs to be counted in the crop len.
let mut remaining_words = crop_size + first_match_word_position - last_match_word_position;
- let mut before_tokens = self.tokens[..first_match_token_position].iter().rev().peekable();
- let mut after_tokens = self.tokens[last_match_token_position..].iter().peekable();
+ let mut before_tokens = tokens[..first_match_token_position].iter().rev().peekable();
+ let mut after_tokens = tokens[last_match_token_position..].iter().peekable();
while remaining_words > 0 {
let before_token = before_tokens.peek().map(|t| t.separator_kind());
@@ -396,7 +403,7 @@ impl<'t> Matcher<'t, '_> {
Cow::Borrowed(self.text)
} else {
match &self.matches {
- Some(matches) => {
+ Some((tokens, matches)) => {
let matches = match format_options.crop {
Some(crop_size) if crop_size > 0 => {
self.find_best_match_interval(matches, crop_size)
@@ -405,7 +412,9 @@ impl<'t> Matcher<'t, '_> {
};
let (byte_start, byte_end) = match format_options.crop {
- Some(crop_size) if crop_size > 0 => self.crop_bounds(matches, crop_size),
+ Some(crop_size) if crop_size > 0 => {
+ self.crop_bounds(tokens, matches, crop_size)
+ }
_ => (0, self.text.len()),
};
@@ -420,7 +429,6 @@ impl<'t> Matcher<'t, '_> {
if format_options.highlight {
// insert highlight markers around matches.
- let tokens = self.tokens;
for m in matches {
let token = &tokens[m.token_position];
@@ -470,7 +478,7 @@ impl<'t> Matcher<'t, '_> {
#[cfg(test)]
mod tests {
- use charabia::Tokenize;
+ use charabia::TokenizerBuilder;
use super::*;
use crate::search::matches::matching_words::MatchingWord;
@@ -485,6 +493,12 @@ mod tests {
MatchingWords::new(matching_words)
}
+ impl MatcherBuilder<'_, Vec> {
+ pub fn from_matching_words(matching_words: MatchingWords) -> Self {
+ Self::new(matching_words, TokenizerBuilder::default().build())
+ }
+ }
+
#[test]
fn format_identity() {
let matching_words = matching_words();
@@ -495,22 +509,22 @@ mod tests {
// Text without any match.
let text = "A quick brown fox can not jump 32 feet, right? Brr, it is cold!";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no crop and no highlight should return complete text.
assert_eq!(&matcher.format(format_options), &text);
// Text containing all matches.
let text = "Natalie risk her future to build a world with the boy she loves. Emily Henry: The Love That Split The World.";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no crop and no highlight should return complete text.
assert_eq!(&matcher.format(format_options), &text);
// Text containing some matches.
let text = "Natalie risk her future to build a world with the boy she loves.";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no crop and no highlight should return complete text.
assert_eq!(&matcher.format(format_options), &text);
}
@@ -525,34 +539,34 @@ mod tests {
// empty text.
let text = "";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
assert_eq!(&matcher.format(format_options), "");
// text containing only separators.
let text = ":-)";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
assert_eq!(&matcher.format(format_options), ":-)");
// Text without any match.
let text = "A quick brown fox can not jump 32 feet, right? Brr, it is cold!";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no crop should return complete text, because there is no matches.
assert_eq!(&matcher.format(format_options), &text);
// Text containing all matches.
let text = "Natalie risk her future to build a world with the boy she loves. Emily Henry: The Love That Split The World.";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no crop should return complete text with highlighted matches.
assert_eq!(&matcher.format(format_options), "Natalie risk her future to build a world with the boy she loves. Emily Henry: The Love That Split The World.");
// Text containing some matches.
let text = "Natalie risk her future to build a world with the boy she loves.";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no crop should return complete text with highlighted matches.
assert_eq!(
&matcher.format(format_options),
@@ -575,22 +589,22 @@ mod tests {
// Text containing prefix match.
let text = "Ŵôřlḑôle";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no crop should return complete text with highlighted matches.
assert_eq!(&matcher.format(format_options), "Ŵôřlḑôle");
// Text containing unicode match.
let text = "Ŵôřlḑ";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no crop should return complete text with highlighted matches.
assert_eq!(&matcher.format(format_options), "Ŵôřlḑ");
// Text containing unicode match.
let text = "Westfália";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no crop should return complete text with highlighted matches.
assert_eq!(&matcher.format(format_options), "Westfália");
}
@@ -605,20 +619,20 @@ mod tests {
// empty text.
let text = "";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
assert_eq!(&matcher.format(format_options), "");
// text containing only separators.
let text = ":-)";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
assert_eq!(&matcher.format(format_options), ":-)");
// Text without any match.
let text = "A quick brown fox can not jump 32 feet, right? Brr, it is cold!";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no highlight should return 10 first words with a marker at the end.
assert_eq!(
&matcher.format(format_options),
@@ -627,8 +641,8 @@ mod tests {
// Text without any match starting by a separator.
let text = "(A quick brown fox can not jump 32 feet, right? Brr, it is cold!)";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no highlight should return 10 first words with a marker at the end.
assert_eq!(
&matcher.format(format_options),
@@ -637,8 +651,8 @@ mod tests {
// Test phrase propagation
let text = "Natalie risk her future. Split The World is a book written by Emily Henry. I never read it.";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// should crop the phrase instead of croping around the match.
assert_eq!(
&matcher.format(format_options),
@@ -647,8 +661,8 @@ mod tests {
// Text containing some matches.
let text = "Natalie risk her future to build a world with the boy she loves.";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no highlight should return 10 last words with a marker at the start.
assert_eq!(
&matcher.format(format_options),
@@ -657,8 +671,8 @@ mod tests {
// Text containing all matches.
let text = "Natalie risk her future to build a world with the boy she loves. Emily Henry: The Love That Split The World.";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// no highlight should return 10 last words with a marker at the start.
assert_eq!(
&matcher.format(format_options),
@@ -667,8 +681,8 @@ mod tests {
// Text containing a match unordered and a match ordered.
let text = "The world split void void void void void void void void void split the world void void";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// crop should return 10 last words with a marker at the start.
assert_eq!(
&matcher.format(format_options),
@@ -677,8 +691,8 @@ mod tests {
// Text containing matches with diferent density.
let text = "split void the void void world void void void void void void void void void void split the world void void";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// crop should return 10 last words with a marker at the start.
assert_eq!(
&matcher.format(format_options),
@@ -687,8 +701,8 @@ mod tests {
// Text containing matches with same word.
let text = "split split split split split split void void void void void void void void void void split the world void void";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// crop should return 10 last words with a marker at the start.
assert_eq!(
&matcher.format(format_options),
@@ -706,20 +720,20 @@ mod tests {
// empty text.
let text = "";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
assert_eq!(&matcher.format(format_options), "");
// text containing only separators.
let text = ":-)";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
assert_eq!(&matcher.format(format_options), ":-)");
// Text without any match.
let text = "A quick brown fox can not jump 32 feet, right? Brr, it is cold!";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// both should return 10 first words with a marker at the end.
assert_eq!(
&matcher.format(format_options),
@@ -728,8 +742,8 @@ mod tests {
// Text containing some matches.
let text = "Natalie risk her future to build a world with the boy she loves.";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// both should return 10 last words with a marker at the start and highlighted matches.
assert_eq!(
&matcher.format(format_options),
@@ -738,15 +752,15 @@ mod tests {
// Text containing all matches.
let text = "Natalie risk her future to build a world with the boy she loves. Emily Henry: The Love That Split The World.";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// both should return 10 last words with a marker at the start and highlighted matches.
assert_eq!(&matcher.format(format_options), "…she loves. Emily Henry: The Love That Split The World.");
// Text containing a match unordered and a match ordered.
let text = "The world split void void void void void void void void void split the world void void";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+
+ let mut matcher = builder.build(text);
// crop should return 10 last words with a marker at the start.
assert_eq!(
&matcher.format(format_options),
@@ -762,26 +776,25 @@ mod tests {
let builder = MatcherBuilder::from_matching_words(matching_words);
let text = "void void split the world void void.";
- let tokens: Vec<_> = text.tokenize().collect();
// set a smaller crop size
let format_options = FormatOptions { highlight: false, crop: Some(2) };
- let mut matcher = builder.build(&tokens[..], text);
+ let mut matcher = builder.build(text);
// because crop size < query size, partially format matches.
assert_eq!(&matcher.format(format_options), "…split the…");
// set a smaller crop size
let format_options = FormatOptions { highlight: false, crop: Some(1) };
- let mut matcher = builder.build(&tokens[..], text);
+ let mut matcher = builder.build(text);
// because crop size < query size, partially format matches.
assert_eq!(&matcher.format(format_options), "…split…");
// set crop size to 0
let format_options = FormatOptions { highlight: false, crop: Some(0) };
- let mut matcher = builder.build(&tokens[..], text);
+ let mut matcher = builder.build(text);
// because crop size is 0, crop is ignored.
assert_eq!(&matcher.format(format_options), "void void split the world void void.");
}
@@ -817,9 +830,8 @@ mod tests {
let format_options = FormatOptions { highlight: true, crop: None };
let text = "the do or die can't be he do and or isn't he";
- let tokens: Vec<_> = text.tokenize().collect();
- let mut matcher = builder.build(&tokens[..], text);
+ let mut matcher = builder.build(text);
assert_eq!(
&matcher.format(format_options),
"_the_ _do_ _or_ die can't be he _do_ and or isn'_t_ _he_",