use std::collections::HashSet; use std::mem; use std::time::Instant; use anyhow::{bail, Context}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; use milli::{Index, obkv_to_json, FacetCondition}; use serde::{Deserialize, Serialize}; use serde_json::{Value, Map}; use crate::index_controller::IndexController; use super::Data; pub const DEFAULT_SEARCH_LIMIT: usize = 20; const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] #[allow(dead_code)] pub struct SearchQuery { pub q: Option, pub offset: Option, #[serde(default = "default_search_limit")] pub limit: usize, pub attributes_to_retrieve: Option>, pub attributes_to_crop: Option>, pub crop_length: Option, pub attributes_to_highlight: Option>, pub filters: Option, pub matches: Option, pub facet_filters: Option, pub facets_distribution: Option>, pub facet_condition: Option, } impl SearchQuery { pub fn perform(&self, index: impl AsRef) -> anyhow::Result{ let index = index.as_ref(); let before_search = Instant::now(); let rtxn = index.read_txn().unwrap(); let mut search = index.search(&rtxn); if let Some(ref query) = self.q { search.query(query); } search.limit(self.limit); search.offset(self.offset.unwrap_or_default()); if let Some(ref condition) = self.facet_condition { if !condition.trim().is_empty() { let condition = FacetCondition::from_str(&rtxn, &index, &condition)?; search.facet_condition(condition); } } let milli::SearchResult { documents_ids, found_words, candidates } = search.execute()?; let mut documents = Vec::new(); let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let displayed_fields = match index.displayed_fields_ids(&rtxn).unwrap() { Some(fields) => fields, None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; let stop_words = fst::Set::default(); let highlighter = Highlighter::new(&stop_words); for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv)?; if let Some(ref attributes_to_highlight) = self.attributes_to_highlight { highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); } documents.push(object); } Ok(SearchResult { hits: documents, nb_hits: candidates.len(), query: self.q.clone().unwrap_or_default(), limit: self.limit, offset: self.offset.unwrap_or_default(), processing_time_ms: before_search.elapsed().as_millis(), }) } } #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct SearchResult { hits: Vec>, nb_hits: u64, query: String, limit: usize, offset: usize, processing_time_ms: u128, } struct Highlighter<'a, A> { analyzer: Analyzer<'a, A>, } impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { fn new(stop_words: &'a fst::Set) -> Self { let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words)); Self { analyzer } } fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { match value { Value::Null => Value::Null, Value::Bool(boolean) => Value::Bool(boolean), Value::Number(number) => Value::Number(number), Value::String(old_string) => { let mut string = String::new(); let analyzed = self.analyzer.analyze(&old_string); for (word, token) in analyzed.reconstruct() { if token.is_word() { let to_highlight = words_to_highlight.contains(token.text()); if to_highlight { string.push_str("") } string.push_str(word); if to_highlight { string.push_str("") } } else { string.push_str(word); } } Value::String(string) }, Value::Array(values) => { Value::Array(values.into_iter() .map(|v| self.highlight_value(v, words_to_highlight)) .collect()) }, Value::Object(object) => { Value::Object(object.into_iter() .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight))) .collect()) }, } } fn highlight_record( &self, object: &mut Map, words_to_highlight: &HashSet, attributes_to_highlight: &HashSet, ) { // TODO do we need to create a string for element that are not and needs to be highlight? for (key, value) in object.iter_mut() { if attributes_to_highlight.contains(key) { let old_value = mem::take(value); *value = self.highlight_value(old_value, words_to_highlight); } } } } impl Data { pub fn search>(&self, index: S, search_query: SearchQuery) -> anyhow::Result { match self.index_controller.index(&index)? { Some(index) => Ok(search_query.perform(index)?), None => bail!("index {:?} doesn't exists", index.as_ref()), } } pub async fn retrieve_documents( &self, index: impl AsRef + Send + Sync + 'static, offset: usize, limit: usize, attributes_to_retrieve: Option>, ) -> anyhow::Result>> where S: AsRef + Send + Sync + 'static { let index_controller = self.index_controller.clone(); let documents: anyhow::Result<_> = tokio::task::spawn_blocking(move || { let index = index_controller .index(&index)? .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; let txn = index.read_txn()?; let fields_ids_map = index.fields_ids_map(&txn)?; let attributes_to_retrieve_ids = match attributes_to_retrieve { Some(attrs) => attrs .iter() .filter_map(|f| fields_ids_map.id(f.as_ref())) .collect::>(), None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; let iter = index.documents.range(&txn, &(..))? .skip(offset) .take(limit); let mut documents = Vec::new(); for entry in iter { let (_id, obkv) = entry?; let object = obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, obkv)?; documents.push(object); } Ok(documents) }).await?; documents } pub async fn retrieve_document( &self, index: impl AsRef + Sync + Send + 'static, document_id: impl AsRef + Sync + Send + 'static, attributes_to_retrieve: Option>, ) -> anyhow::Result> where S: AsRef + Sync + Send + 'static, { let index_controller = self.index_controller.clone(); let document: anyhow::Result<_> = tokio::task::spawn_blocking(move || { let index = index_controller .index(&index)? .with_context(|| format!("Index {:?} doesn't exist", index.as_ref()))?; let txn = index.read_txn()?; let fields_ids_map = index.fields_ids_map(&txn)?; let attributes_to_retrieve_ids = match attributes_to_retrieve { Some(attrs) => attrs .iter() .filter_map(|f| fields_ids_map.id(f.as_ref())) .collect::>(), None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; let internal_id = index .external_documents_ids(&txn)? .get(document_id.as_ref().as_bytes()) .with_context(|| format!("Document with id {} not found", document_id.as_ref()))?; let document = index.documents(&txn, std::iter::once(internal_id))? .into_iter() .next() .map(|(_, d)| d); match document { Some(document) => Ok(obkv_to_json(&attributes_to_retrieve_ids, &fields_ids_map, document)?), None => bail!("Document with id {} not found", document_id.as_ref()), } }).await?; document } }