snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-25 11:35:05 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Fix positional databases

Browse Source
This commit is contained in:
ManyTheFish 2024-11-14 11:40:12 +01:00
parent 9e8367f1e6
commit 91c58cfa38
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
crates/milli/src/update/new/extract/searchable/tokenize_document.rs
View File

@ -3,7 +3,6 @@ use std::collections::HashMap;
use charabia::{SeparatorKind, Token, TokenKind, Tokenizer, TokenizerBuilder};
use serde_json::Value;
use crate::proximity::MAX_DISTANCE;
use crate::update::new::document::Document;
use crate::update::new::extract::perm_json_p::{
seek_leaf_values_in_array, seek_leaf_values_in_object, select_field,
@ -13,6 +12,9 @@ use crate::{
MAX_WORD_LENGTH,
};
// todo: should be crate::proximity::MAX_DISTANCE but it has been forgotten
const MAX_DISTANCE: u32 = 8;
pub struct DocumentTokenizer<'a> {
pub tokenizer: &'a Tokenizer<'a>,
pub attribute_to_extract: Option<&'a [&'a str]>,
@ -251,22 +253,22 @@ mod test {
]: "doggo",
[
2,
MAX_DISTANCE,
8,
]: "doggo",
[
2,
16,
]: "catto",
[
3,
5,
0,
]: "10",
[
4,
6,
0,
]: "pesti",
[
5,
7,
0,
]: "23",
}