From fe69385bd75ff90d15889a2d5d675cfc00b825ba Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Tue, 3 Sep 2024 14:24:37 +0200 Subject: [PATCH] Fix tokenizer test --- milli/src/update/new/extract/tokenize_document.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/milli/src/update/new/extract/tokenize_document.rs b/milli/src/update/new/extract/tokenize_document.rs index 9f0a1c4d8..1494dd4b2 100644 --- a/milli/src/update/new/extract/tokenize_document.rs +++ b/milli/src/update/new/extract/tokenize_document.rs @@ -270,10 +270,6 @@ mod test { let value = obkv.into_inner().unwrap(); let obkv = KvReader::from_slice(value.as_slice()); - fields_ids_map.insert("doggo.age"); - fields_ids_map.insert("catto.catto.name"); - fields_ids_map.insert("catto.catto.age"); - let mut tb = TokenizerBuilder::default(); let document_tokenizer = DocumentTokenizer { tokenizer: &tb.build(), @@ -282,9 +278,12 @@ mod test { max_positions_per_attributes: 1000, }; + let fields_ids_map_lock = std::sync::RwLock::new(fields_ids_map); + let mut global_fields_ids_map = GlobalFieldsIdsMap::new(&fields_ids_map_lock); + let mut words = std::collections::BTreeMap::new(); document_tokenizer - .tokenize_document(obkv, &fields_ids_map, &mut |fid, pos, word| { + .tokenize_document(obkv, &mut global_fields_ids_map, &mut |fid, pos, word| { words.insert([fid, pos], word.to_string()); }) .unwrap();