snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 10:07:40 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Fix positional databases

Browse Source
This commit is contained in:
ManyTheFish 2024-11-14 11:40:12 +01:00
parent 9e8367f1e6
commit 91c58cfa38
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
crates/milli/src/update/new/extract/searchable/tokenize_document.rs
View File

@ -3,7 +3,6 @@ use std::collections::HashMap;
use charabia::{SeparatorKind, Token, TokenKind, Tokenizer, TokenizerBuilder}; use charabia::{SeparatorKind, Token, TokenKind, Tokenizer, TokenizerBuilder};
use serde_json::Value; use serde_json::Value;
use crate::proximity::MAX_DISTANCE;
use crate::update::new::document::Document; use crate::update::new::document::Document;
use crate::update::new::extract::perm_json_p::{ use crate::update::new::extract::perm_json_p::{
seek_leaf_values_in_array, seek_leaf_values_in_object, select_field, seek_leaf_values_in_array, seek_leaf_values_in_object, select_field,
@ -13,6 +12,9 @@ use crate::{
MAX_WORD_LENGTH, MAX_WORD_LENGTH,
}; };
// todo: should be crate::proximity::MAX_DISTANCE but it has been forgotten
const MAX_DISTANCE: u32 = 8;
pub struct DocumentTokenizer<'a> { pub struct DocumentTokenizer<'a> {
pub tokenizer: &'a Tokenizer<'a>, pub tokenizer: &'a Tokenizer<'a>,
pub attribute_to_extract: Option<&'a [&'a str]>, pub attribute_to_extract: Option<&'a [&'a str]>,
@ -251,22 +253,22 @@ mod test {
]: "doggo", ]: "doggo",
[ [
2, 2,
MAX_DISTANCE, 8,
]: "doggo", ]: "doggo",
[ [
2, 2,
16, 16,
]: "catto", ]: "catto",
[ [
3, 5,
0, 0,
]: "10", ]: "10",
[ [
4, 6,
0, 0,
]: "pesti", ]: "pesti",
[ [
5, 7,
0, 0,
]: "23", ]: "23",
} }