From bbe1845f668f377fe2c451fef08fbbfb1bdc6859 Mon Sep 17 00:00:00 2001 From: qdequele Date: Fri, 10 Jan 2020 18:20:30 +0100 Subject: [PATCH] squash-me --- .../src/criterion/sort_by_attr.rs | 16 +- meilisearch-core/src/error.rs | 1 + meilisearch-core/src/fields_map.rs | 94 ++ meilisearch-core/src/lib.rs | 4 +- meilisearch-core/src/ranked_map.rs | 10 +- meilisearch-core/src/raw_indexer.rs | 20 +- meilisearch-core/src/serde/deserializer.rs | 20 +- meilisearch-core/src/serde/indexer.rs | 32 +- meilisearch-core/src/serde/mod.rs | 9 + meilisearch-core/src/serde/serializer.rs | 61 +- .../src/store/documents_fields_counts.rs | 22 +- meilisearch-core/src/store/main.rs | 12 + meilisearch-core/src/store/mod.rs | 7 +- .../src/update/documents_addition.rs | 8 + meilisearch-http/src/routes/stop_words.rs | 24 +- meilisearch-http/src/routes/synonym.rs | 22 +- meilisearch-schema/src/error.rs | 20 + meilisearch-schema/src/fields_map.rs | 91 ++ meilisearch-schema/src/lib.rs | 1180 +++++++++-------- meilisearch-schema/src/schema.rs | 141 ++ 20 files changed, 1118 insertions(+), 676 deletions(-) create mode 100644 meilisearch-core/src/fields_map.rs create mode 100644 meilisearch-schema/src/error.rs create mode 100644 meilisearch-schema/src/fields_map.rs create mode 100644 meilisearch-schema/src/schema.rs diff --git a/meilisearch-core/src/criterion/sort_by_attr.rs b/meilisearch-core/src/criterion/sort_by_attr.rs index 3fd801550..3f2fb9461 100644 --- a/meilisearch-core/src/criterion/sort_by_attr.rs +++ b/meilisearch-core/src/criterion/sort_by_attr.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; use std::error::Error; use std::fmt; -use meilisearch_schema::{Schema, SchemaAttr}; +use meilisearch_schema::{Schema, FieldId}; use crate::{RankedMap, RawDocument}; use super::{Criterion, Context}; @@ -41,7 +41,7 @@ use super::{Criterion, Context}; /// ``` pub struct SortByAttr<'a> { ranked_map: &'a RankedMap, - attr: SchemaAttr, + field_id: FieldId, reversed: bool, } @@ -68,18 +68,18 @@ impl<'a> SortByAttr<'a> { attr_name: &str, reversed: bool, ) -> Result, SortByAttrError> { - let attr = match schema.attribute(attr_name) { - Some(attr) => attr, + let field_id = match schema.get_id(attr_name) { + Some(field_id) => *field_id, None => return Err(SortByAttrError::AttributeNotFound), }; - if !schema.props(attr).is_ranked() { + if !schema.id_is_ranked(field_id) { return Err(SortByAttrError::AttributeNotRegisteredForRanking); } Ok(SortByAttr { ranked_map, - attr, + field_id, reversed, }) } @@ -91,8 +91,8 @@ impl Criterion for SortByAttr<'_> { } fn evaluate(&self, _ctx: &Context, lhs: &RawDocument, rhs: &RawDocument) -> Ordering { - let lhs = self.ranked_map.get(lhs.id, self.attr); - let rhs = self.ranked_map.get(rhs.id, self.attr); + let lhs = self.ranked_map.get(lhs.id, self.field_id); + let rhs = self.ranked_map.get(rhs.id, self.field_id); match (lhs, rhs) { (Some(lhs), Some(rhs)) => { diff --git a/meilisearch-core/src/error.rs b/meilisearch-core/src/error.rs index 7d58e2756..5ebd1c947 100644 --- a/meilisearch-core/src/error.rs +++ b/meilisearch-core/src/error.rs @@ -12,6 +12,7 @@ pub enum Error { SchemaMissing, WordIndexMissing, MissingDocumentId, + MaxFieldsLimitExceeded, Zlmdb(heed::Error), Fst(fst::Error), SerdeJson(SerdeJsonError), diff --git a/meilisearch-core/src/fields_map.rs b/meilisearch-core/src/fields_map.rs new file mode 100644 index 000000000..cca52bc46 --- /dev/null +++ b/meilisearch-core/src/fields_map.rs @@ -0,0 +1,94 @@ +use std::io::{Read, Write}; +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use crate::{MResult, Error}; + + +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FieldsMap { + name_map: HashMap, + id_map: HashMap, + next_id: u16 +} + +impl FieldsMap { + pub fn len(&self) -> usize { + self.name_map.len() + } + + pub fn is_empty(&self) -> bool { + self.name_map.is_empty() + } + + pub fn insert(&mut self, name: T) -> MResult { + let name = name.to_string(); + if let Some(id) = self.name_map.get(&name) { + return Ok(*id) + } + let id = self.next_id; + if self.next_id.checked_add(1).is_none() { + return Err(Error::MaxFieldsLimitExceeded) + } else { + self.next_id += 1; + } + self.name_map.insert(name.clone(), id); + self.id_map.insert(id, name); + Ok(id) + } + + pub fn remove(&mut self, name: T) { + let name = name.to_string(); + if let Some(id) = self.name_map.get(&name) { + self.id_map.remove(&id); + } + self.name_map.remove(&name); + } + + pub fn get_id(&self, name: T) -> Option<&u16> { + let name = name.to_string(); + self.name_map.get(&name) + } + + pub fn get_name(&self, id: u16) -> Option<&String> { + self.id_map.get(&id) + } + + pub fn read_from_bin(reader: R) -> bincode::Result { + bincode::deserialize_from(reader) + } + + pub fn write_to_bin(&self, writer: W) -> bincode::Result<()> { + bincode::serialize_into(writer, &self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fields_map() { + let mut fields_map = FieldsMap::default(); + + assert_eq!(fields_map.insert("id").unwrap(), 0); + assert_eq!(fields_map.insert("title").unwrap(), 1); + assert_eq!(fields_map.insert("descritpion").unwrap(), 2); + assert_eq!(fields_map.insert("id").unwrap(), 0); + assert_eq!(fields_map.insert("title").unwrap(), 1); + assert_eq!(fields_map.insert("descritpion").unwrap(), 2); + assert_eq!(fields_map.get_id("id"), Some(&0)); + assert_eq!(fields_map.get_id("title"), Some(&1)); + assert_eq!(fields_map.get_id("descritpion"), Some(&2)); + assert_eq!(fields_map.get_id("date"), None); + assert_eq!(fields_map.len(), 3); + assert_eq!(fields_map.get_name(0), Some(&"id".to_owned())); + assert_eq!(fields_map.get_name(1), Some(&"title".to_owned())); + assert_eq!(fields_map.get_name(2), Some(&"descritpion".to_owned())); + assert_eq!(fields_map.get_name(4), None); + fields_map.remove("title"); + assert_eq!(fields_map.get_id("title"), None); + assert_eq!(fields_map.insert("title").unwrap(), 3); + assert_eq!(fields_map.len(), 3); + } +} diff --git a/meilisearch-core/src/lib.rs b/meilisearch-core/src/lib.rs index 8c15a6c23..ca4714ed8 100644 --- a/meilisearch-core/src/lib.rs +++ b/meilisearch-core/src/lib.rs @@ -16,7 +16,8 @@ mod ranked_map; mod raw_document; mod reordered_attrs; mod update; -mod settings; +// mod fields_map; +pub mod settings; pub mod criterion; pub mod raw_indexer; pub mod serde; @@ -26,6 +27,7 @@ pub use self::database::{BoxUpdateFn, Database, MainT, UpdateT}; pub use self::error::{Error, MResult}; pub use self::number::{Number, ParseNumberError}; pub use self::ranked_map::RankedMap; +// pub use self::fields_map::FieldsMap; pub use self::raw_document::RawDocument; pub use self::store::Index; pub use self::update::{EnqueuedUpdateResult, ProcessedUpdateResult, UpdateStatus, UpdateType}; diff --git a/meilisearch-core/src/ranked_map.rs b/meilisearch-core/src/ranked_map.rs index b50cf6ffb..964e37375 100644 --- a/meilisearch-core/src/ranked_map.rs +++ b/meilisearch-core/src/ranked_map.rs @@ -1,14 +1,14 @@ use std::io::{Read, Write}; use hashbrown::HashMap; -use meilisearch_schema::SchemaAttr; +use meilisearch_schema::FieldId; use serde::{Deserialize, Serialize}; use crate::{DocumentId, Number}; #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(transparent)] -pub struct RankedMap(HashMap<(DocumentId, SchemaAttr), Number>); +pub struct RankedMap(HashMap<(DocumentId, FieldId), Number>); impl RankedMap { pub fn len(&self) -> usize { @@ -19,15 +19,15 @@ impl RankedMap { self.0.is_empty() } - pub fn insert(&mut self, document: DocumentId, attribute: SchemaAttr, number: Number) { + pub fn insert(&mut self, document: DocumentId, attribute: FieldId, number: Number) { self.0.insert((document, attribute), number); } - pub fn remove(&mut self, document: DocumentId, attribute: SchemaAttr) { + pub fn remove(&mut self, document: DocumentId, attribute: FieldId) { self.0.remove(&(document, attribute)); } - pub fn get(&self, document: DocumentId, attribute: SchemaAttr) -> Option { + pub fn get(&self, document: DocumentId, attribute: FieldId) -> Option { self.0.get(&(document, attribute)).cloned() } diff --git a/meilisearch-core/src/raw_indexer.rs b/meilisearch-core/src/raw_indexer.rs index bf37fe3b3..b573300cd 100644 --- a/meilisearch-core/src/raw_indexer.rs +++ b/meilisearch-core/src/raw_indexer.rs @@ -3,7 +3,7 @@ use std::convert::TryFrom; use crate::{DocIndex, DocumentId}; use deunicode::deunicode_with_tofu; -use meilisearch_schema::SchemaAttr; +use meilisearch_schema::IndexedPos; use meilisearch_tokenizer::{is_cjk, SeqTokenizer, Token, Tokenizer}; use sdset::SetBuf; @@ -37,14 +37,14 @@ impl RawIndexer { } } - pub fn index_text(&mut self, id: DocumentId, attr: SchemaAttr, text: &str) -> usize { + pub fn index_text(&mut self, id: DocumentId, indexed_pos: IndexedPos, text: &str) -> usize { let mut number_of_words = 0; for token in Tokenizer::new(text) { let must_continue = index_token( token, id, - attr, + indexed_pos, self.word_limit, &self.stop_words, &mut self.words_doc_indexes, @@ -61,7 +61,7 @@ impl RawIndexer { number_of_words } - pub fn index_text_seq<'a, I>(&mut self, id: DocumentId, attr: SchemaAttr, iter: I) + pub fn index_text_seq<'a, I>(&mut self, id: DocumentId, indexed_pos: IndexedPos, iter: I) where I: IntoIterator, { @@ -70,7 +70,7 @@ impl RawIndexer { let must_continue = index_token( token, id, - attr, + indexed_pos, self.word_limit, &self.stop_words, &mut self.words_doc_indexes, @@ -110,7 +110,7 @@ impl RawIndexer { fn index_token( token: Token, id: DocumentId, - attr: SchemaAttr, + indexed_pos: IndexedPos, word_limit: usize, stop_words: &fst::Set, words_doc_indexes: &mut BTreeMap>, @@ -127,7 +127,7 @@ fn index_token( }; if !stop_words.contains(&token.word) { - match token_to_docindex(id, attr, token) { + match token_to_docindex(id, indexed_pos, token) { Some(docindex) => { let word = Vec::from(token.word); @@ -160,14 +160,14 @@ fn index_token( true } -fn token_to_docindex(id: DocumentId, attr: SchemaAttr, token: Token) -> Option { +fn token_to_docindex(id: DocumentId, indexed_pos: IndexedPos, token: Token) -> Option { let word_index = u16::try_from(token.word_index).ok()?; let char_index = u16::try_from(token.char_index).ok()?; let char_length = u16::try_from(token.word.chars().count()).ok()?; let docindex = DocIndex { document_id: id, - attribute: attr.0, + attribute: indexed_pos.0, word_index, char_index, char_length, @@ -178,7 +178,9 @@ fn token_to_docindex(id: DocumentId, attr: SchemaAttr, token: Token) -> Option { pub reader: &'a heed::RoTxn, pub documents_fields: DocumentsFields, pub schema: &'a Schema, - pub attributes: Option<&'a HashSet>, + pub attributes: Option<&'a HashSet>, } impl<'de, 'a, 'b> de::Deserializer<'de> for &'b mut Deserializer<'a> { @@ -92,15 +92,17 @@ impl<'de, 'a, 'b> de::Deserializer<'de> for &'b mut Deserializer<'a> { } }; - let is_displayed = self.schema.props(attr).is_displayed(); + let is_displayed = self.schema.id_is_displayed(attr); if is_displayed && self.attributes.map_or(true, |f| f.contains(&attr)) { - let attribute_name = self.schema.attribute_name(attr); + if let Some(attribute_name) = self.schema.get_name(attr) { + let cursor = Cursor::new(value.to_owned()); + let ioread = SerdeJsonIoRead::new(cursor); + let value = Value(SerdeJsonDeserializer::new(ioread)); - let cursor = Cursor::new(value.to_owned()); - let ioread = SerdeJsonIoRead::new(cursor); - let value = Value(SerdeJsonDeserializer::new(ioread)); - - Some((attribute_name, value)) + Some((*attribute_name, value)) + } else { + None + } } else { None } diff --git a/meilisearch-core/src/serde/indexer.rs b/meilisearch-core/src/serde/indexer.rs index 62ee28248..67599ef7f 100644 --- a/meilisearch-core/src/serde/indexer.rs +++ b/meilisearch-core/src/serde/indexer.rs @@ -1,4 +1,4 @@ -use meilisearch_schema::SchemaAttr; +use meilisearch_schema::{IndexedPos}; use serde::ser; use serde::Serialize; @@ -7,7 +7,7 @@ use crate::raw_indexer::RawIndexer; use crate::DocumentId; pub struct Indexer<'a> { - pub attribute: SchemaAttr, + pub pos: IndexedPos, pub indexer: &'a mut RawIndexer, pub document_id: DocumentId, } @@ -85,7 +85,7 @@ impl<'a> ser::Serializer for Indexer<'a> { fn serialize_str(self, text: &str) -> Result { let number_of_words = self .indexer - .index_text(self.document_id, self.attribute, text); + .index_text(self.document_id, self.pos, text); Ok(Some(number_of_words)) } @@ -104,7 +104,7 @@ impl<'a> ser::Serializer for Indexer<'a> { let text = value.serialize(ConvertToString)?; let number_of_words = self .indexer - .index_text(self.document_id, self.attribute, &text); + .index_text(self.document_id, self.pos, &text); Ok(Some(number_of_words)) } @@ -153,7 +153,7 @@ impl<'a> ser::Serializer for Indexer<'a> { fn serialize_seq(self, _len: Option) -> Result { let indexer = SeqIndexer { - attribute: self.attribute, + pos: self.pos, document_id: self.document_id, indexer: self.indexer, texts: Vec::new(), @@ -164,7 +164,7 @@ impl<'a> ser::Serializer for Indexer<'a> { fn serialize_tuple(self, _len: usize) -> Result { let indexer = TupleIndexer { - attribute: self.attribute, + pos: self.pos, document_id: self.document_id, indexer: self.indexer, texts: Vec::new(), @@ -197,7 +197,7 @@ impl<'a> ser::Serializer for Indexer<'a> { fn serialize_map(self, _len: Option) -> Result { let indexer = MapIndexer { - attribute: self.attribute, + pos: self.pos, document_id: self.document_id, indexer: self.indexer, texts: Vec::new(), @@ -212,7 +212,7 @@ impl<'a> ser::Serializer for Indexer<'a> { _len: usize, ) -> Result { let indexer = StructIndexer { - attribute: self.attribute, + pos: self.pos, document_id: self.document_id, indexer: self.indexer, texts: Vec::new(), @@ -235,7 +235,7 @@ impl<'a> ser::Serializer for Indexer<'a> { } pub struct SeqIndexer<'a> { - attribute: SchemaAttr, + pos: IndexedPos, document_id: DocumentId, indexer: &'a mut RawIndexer, texts: Vec, @@ -257,13 +257,13 @@ impl<'a> ser::SerializeSeq for SeqIndexer<'a> { fn end(self) -> Result { let texts = self.texts.iter().map(String::as_str); self.indexer - .index_text_seq(self.document_id, self.attribute, texts); + .index_text_seq(self.document_id, self.pos, texts); Ok(None) } } pub struct MapIndexer<'a> { - attribute: SchemaAttr, + pos: IndexedPos, document_id: DocumentId, indexer: &'a mut RawIndexer, texts: Vec, @@ -294,13 +294,13 @@ impl<'a> ser::SerializeMap for MapIndexer<'a> { fn end(self) -> Result { let texts = self.texts.iter().map(String::as_str); self.indexer - .index_text_seq(self.document_id, self.attribute, texts); + .index_text_seq(self.document_id, self.pos, texts); Ok(None) } } pub struct StructIndexer<'a> { - attribute: SchemaAttr, + pos: IndexedPos, document_id: DocumentId, indexer: &'a mut RawIndexer, texts: Vec, @@ -328,13 +328,13 @@ impl<'a> ser::SerializeStruct for StructIndexer<'a> { fn end(self) -> Result { let texts = self.texts.iter().map(String::as_str); self.indexer - .index_text_seq(self.document_id, self.attribute, texts); + .index_text_seq(self.document_id, self.pos, texts); Ok(None) } } pub struct TupleIndexer<'a> { - attribute: SchemaAttr, + pos: IndexedPos, document_id: DocumentId, indexer: &'a mut RawIndexer, texts: Vec, @@ -356,7 +356,7 @@ impl<'a> ser::SerializeTuple for TupleIndexer<'a> { fn end(self) -> Result { let texts = self.texts.iter().map(String::as_str); self.indexer - .index_text_seq(self.document_id, self.attribute, texts); + .index_text_seq(self.document_id, self.pos, texts); Ok(None) } } diff --git a/meilisearch-core/src/serde/mod.rs b/meilisearch-core/src/serde/mod.rs index 0469f352d..0e2d58a2c 100644 --- a/meilisearch-core/src/serde/mod.rs +++ b/meilisearch-core/src/serde/mod.rs @@ -26,6 +26,7 @@ use std::{error::Error, fmt}; use serde::ser; use serde_json::Error as SerdeJsonError; +use meilisearch_schema::Error as SchemaError; use crate::ParseNumberError; @@ -36,6 +37,7 @@ pub enum SerializerError { Zlmdb(heed::Error), SerdeJson(SerdeJsonError), ParseNumber(ParseNumberError), + Schema(SchemaError), UnserializableType { type_name: &'static str }, UnindexableType { type_name: &'static str }, UnrankableType { type_name: &'static str }, @@ -62,6 +64,7 @@ impl fmt::Display for SerializerError { SerializerError::ParseNumber(e) => { write!(f, "error while trying to parse a number: {}", e) } + SerializerError::Schema(e) => write!(f, "impossible to update schema: {}", e), SerializerError::UnserializableType { type_name } => { write!(f, "{} is not a serializable type", type_name) } @@ -101,3 +104,9 @@ impl From for SerializerError { SerializerError::ParseNumber(error) } } + +impl From for SerializerError { + fn from(error: SchemaError) -> SerializerError { + SerializerError::Schema(error) + } +} diff --git a/meilisearch-core/src/serde/serializer.rs b/meilisearch-core/src/serde/serializer.rs index 2016cd314..cf3be929b 100644 --- a/meilisearch-core/src/serde/serializer.rs +++ b/meilisearch-core/src/serde/serializer.rs @@ -1,4 +1,4 @@ -use meilisearch_schema::{Schema, SchemaAttr, SchemaProps}; +use meilisearch_schema::{Schema, FieldsMap}; use serde::ser; use crate::database::MainT; @@ -15,6 +15,7 @@ pub struct Serializer<'a, 'b> { pub document_fields_counts: DocumentsFieldsCounts, pub indexer: &'a mut RawIndexer, pub ranked_map: &'a mut RankedMap, + pub fields_map: &'a mut FieldsMap, pub document_id: DocumentId, } @@ -158,6 +159,7 @@ impl<'a, 'b> ser::Serializer for Serializer<'a, 'b> { document_fields_counts: self.document_fields_counts, indexer: self.indexer, ranked_map: self.ranked_map, + fields_map: self.fields_map, current_key_name: None, }) } @@ -175,6 +177,7 @@ impl<'a, 'b> ser::Serializer for Serializer<'a, 'b> { document_fields_counts: self.document_fields_counts, indexer: self.indexer, ranked_map: self.ranked_map, + fields_map: self.fields_map, }) } @@ -199,6 +202,7 @@ pub struct MapSerializer<'a, 'b> { document_fields_counts: DocumentsFieldsCounts, indexer: &'a mut RawIndexer, ranked_map: &'a mut RankedMap, + fields_map: &'a mut FieldsMap, current_key_name: Option, } @@ -243,6 +247,7 @@ impl<'a, 'b> ser::SerializeMap for MapSerializer<'a, 'b> { self.document_fields_counts, self.indexer, self.ranked_map, + self.fields_map, value, ), None => Ok(()), @@ -262,6 +267,7 @@ pub struct StructSerializer<'a, 'b> { document_fields_counts: DocumentsFieldsCounts, indexer: &'a mut RawIndexer, ranked_map: &'a mut RankedMap, + fields_map: &'a mut FieldsMap, } impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> { @@ -276,20 +282,26 @@ impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> { where T: ser::Serialize, { - match self.schema.attribute(key) { - Some(attribute) => serialize_value( - self.txn, - attribute, - self.schema.props(attribute), - self.document_id, - self.document_store, - self.document_fields_counts, - self.indexer, - self.ranked_map, - value, - ), - None => Ok(()), - } + // let id = fields_map.insert(key)?; + + // let attribute = match self.schema.attribute(id) { + // Some(attribute) => attribute, + // None => { + + // }, + // } + + serialize_value( + self.txn, + attribute, + self.schema.props(attribute), + self.document_id, + self.document_store, + self.document_fields_counts, + self.indexer, + self.ranked_map, + value, + ) } fn end(self) -> Result { @@ -297,10 +309,10 @@ impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> { } } -pub fn serialize_value( +pub fn serialize_value<'a, T: ?Sized>( txn: &mut heed::RwTxn, - attribute: SchemaAttr, - props: SchemaProps, + attribute: &'static str, + schema: &'a Schema, document_id: DocumentId, document_store: DocumentsFields, documents_fields_counts: DocumentsFieldsCounts, @@ -312,11 +324,12 @@ where T: ser::Serialize, { let serialized = serde_json::to_vec(value)?; - document_store.put_document_field(txn, document_id, attribute, &serialized)?; + let field_id = schema.get_or_create(attribute)?; + document_store.put_document_field(txn, document_id, field_id, &serialized)?; - if props.is_indexed() { + if let Some(indexed_pos) = schema.id_is_indexed(field_id) { let indexer = Indexer { - attribute, + field_id, indexer, document_id, }; @@ -324,15 +337,15 @@ where documents_fields_counts.put_document_field_count( txn, document_id, - attribute, + field_id, number_of_words as u16, )?; } } - if props.is_ranked() { + if let Some(field_id) = schema.id_is_ranked(field_id) { let number = value.serialize(ConvertToNumber)?; - ranked_map.insert(document_id, attribute, number); + ranked_map.insert(document_id, field_id, number); } Ok(()) diff --git a/meilisearch-core/src/store/documents_fields_counts.rs b/meilisearch-core/src/store/documents_fields_counts.rs index 0a7eb1bbf..87e3e8fb0 100644 --- a/meilisearch-core/src/store/documents_fields_counts.rs +++ b/meilisearch-core/src/store/documents_fields_counts.rs @@ -3,7 +3,7 @@ use crate::database::MainT; use crate::DocumentId; use heed::types::OwnedType; use heed::Result as ZResult; -use meilisearch_schema::SchemaAttr; +use meilisearch_schema::FieldId; #[derive(Copy, Clone)] pub struct DocumentsFieldsCounts { @@ -15,7 +15,7 @@ impl DocumentsFieldsCounts { self, writer: &mut heed::RwTxn, document_id: DocumentId, - attribute: SchemaAttr, + attribute: FieldId, value: u16, ) -> ZResult<()> { let key = DocumentAttrKey::new(document_id, attribute); @@ -27,8 +27,8 @@ impl DocumentsFieldsCounts { writer: &mut heed::RwTxn, document_id: DocumentId, ) -> ZResult { - let start = DocumentAttrKey::new(document_id, SchemaAttr::min()); - let end = DocumentAttrKey::new(document_id, SchemaAttr::max()); + let start = DocumentAttrKey::new(document_id, FieldId::min()); + let end = DocumentAttrKey::new(document_id, FieldId::max()); self.documents_fields_counts .delete_range(writer, &(start..=end)) } @@ -41,7 +41,7 @@ impl DocumentsFieldsCounts { self, reader: &heed::RoTxn, document_id: DocumentId, - attribute: SchemaAttr, + attribute: FieldId, ) -> ZResult> { let key = DocumentAttrKey::new(document_id, attribute); match self.documents_fields_counts.get(reader, &key)? { @@ -55,8 +55,8 @@ impl DocumentsFieldsCounts { reader: &'txn heed::RoTxn, document_id: DocumentId, ) -> ZResult> { - let start = DocumentAttrKey::new(document_id, SchemaAttr::min()); - let end = DocumentAttrKey::new(document_id, SchemaAttr::max()); + let start = DocumentAttrKey::new(document_id, FieldId::min()); + let end = DocumentAttrKey::new(document_id, FieldId::max()); let iter = self.documents_fields_counts.range(reader, &(start..=end))?; Ok(DocumentFieldsCountsIter { iter }) } @@ -83,12 +83,12 @@ pub struct DocumentFieldsCountsIter<'txn> { } impl Iterator for DocumentFieldsCountsIter<'_> { - type Item = ZResult<(SchemaAttr, u16)>; + type Item = ZResult<(FieldId, u16)>; fn next(&mut self) -> Option { match self.iter.next() { Some(Ok((key, count))) => { - let attr = SchemaAttr(key.attr.get()); + let attr = FieldId(key.attr.get()); Some(Ok((attr, count))) } Some(Err(e)) => Some(Err(e)), @@ -127,13 +127,13 @@ pub struct AllDocumentsFieldsCountsIter<'txn> { } impl Iterator for AllDocumentsFieldsCountsIter<'_> { - type Item = ZResult<(DocumentId, SchemaAttr, u16)>; + type Item = ZResult<(DocumentId, FieldId, u16)>; fn next(&mut self) -> Option { match self.iter.next() { Some(Ok((key, count))) => { let docid = DocumentId(key.docid.get()); - let attr = SchemaAttr(key.attr.get()); + let attr = FieldId(key.attr.get()); Some(Ok((docid, attr, count))) } Some(Err(e)) => Some(Err(e)), diff --git a/meilisearch-core/src/store/main.rs b/meilisearch-core/src/store/main.rs index f2e7a6b3c..7eb1c73a1 100644 --- a/meilisearch-core/src/store/main.rs +++ b/meilisearch-core/src/store/main.rs @@ -1,3 +1,4 @@ +use crate::fields_map::FieldsMap; use crate::database::MainT; use crate::RankedMap; use chrono::{DateTime, Utc}; @@ -17,6 +18,7 @@ const FIELDS_FREQUENCY_KEY: &str = "fields-frequency"; const NAME_KEY: &str = "name"; const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents"; const RANKED_MAP_KEY: &str = "ranked-map"; +const FIELDS_MAP_KEY: &str = "fields-map"; const SCHEMA_KEY: &str = "schema"; const UPDATED_AT_KEY: &str = "updated-at"; const WORDS_KEY: &str = "words"; @@ -112,6 +114,16 @@ impl Main { .get::<_, Str, SerdeBincode>(reader, RANKED_MAP_KEY) } + pub fn put_fields_map(self, writer: &mut heed::RwTxn, fields_map: &FieldsMap) -> ZResult<()> { + self.main + .put::<_, Str, SerdeBincode>(writer, FIELDS_MAP_KEY, &fields_map) + } + + pub fn fields_map(self, reader: &heed::RoTxn) -> ZResult> { + self.main + .get::<_, Str, SerdeBincode>(reader, FIELDS_MAP_KEY) + } + pub fn put_synonyms_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> { let bytes = fst.as_fst().as_bytes(); self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes) diff --git a/meilisearch-core/src/store/mod.rs b/meilisearch-core/src/store/mod.rs index 98e9fcab7..5aa26e1ea 100644 --- a/meilisearch-core/src/store/mod.rs +++ b/meilisearch-core/src/store/mod.rs @@ -206,11 +206,10 @@ impl Index { let schema = self.main.schema(reader)?; let schema = schema.ok_or(Error::SchemaMissing)?; + // let attributes = attributes.map(|a| a.iter().filter_map(|name| schema.get_id(*name)).collect()); + let attributes = match attributes { - Some(attributes) => attributes - .iter() - .map(|name| schema.attribute(name)) - .collect(), + Some(attributes) => Some(attributes.iter().filter_map(|name| schema.get_id(*name)).collect()), None => None, }; diff --git a/meilisearch-core/src/update/documents_addition.rs b/meilisearch-core/src/update/documents_addition.rs index 5c14000b8..0431b0fbc 100644 --- a/meilisearch-core/src/update/documents_addition.rs +++ b/meilisearch-core/src/update/documents_addition.rs @@ -147,6 +147,8 @@ pub fn apply_documents_addition<'a, 'b>( None => fst::Set::default(), }; + let mut fields_map = main_store.fields_map(writer)?.unwrap_or_default(); + // 3. index the documents fields in the stores let mut indexer = RawIndexer::new(stop_words); @@ -158,6 +160,7 @@ pub fn apply_documents_addition<'a, 'b>( document_fields_counts: index.documents_fields_counts, indexer: &mut indexer, ranked_map: &mut ranked_map, + fields_map: &mut fields_map, document_id, }; @@ -238,6 +241,8 @@ pub fn apply_documents_partial_addition<'a, 'b>( None => fst::Set::default(), }; + let mut fields_map = main_store.fields_map(writer)?.unwrap_or_default(); + // 3. index the documents fields in the stores let mut indexer = RawIndexer::new(stop_words); @@ -249,6 +254,7 @@ pub fn apply_documents_partial_addition<'a, 'b>( document_fields_counts: index.documents_fields_counts, indexer: &mut indexer, ranked_map: &mut ranked_map, + fields_map: &mut fields_map, document_id, }; @@ -275,6 +281,7 @@ pub fn reindex_all_documents(writer: &mut heed::RwTxn, index: &store::Ind }; let mut ranked_map = RankedMap::default(); + let mut fields_map = main_store.fields_map(writer)?.unwrap_or_default(); // 1. retrieve all documents ids let mut documents_ids_to_reindex = Vec::new(); @@ -318,6 +325,7 @@ pub fn reindex_all_documents(writer: &mut heed::RwTxn, index: &store::Ind index.documents_fields_counts, &mut indexer, &mut ranked_map, + &mut fields_map, &value, )?; } diff --git a/meilisearch-http/src/routes/stop_words.rs b/meilisearch-http/src/routes/stop_words.rs index 04f401523..c654b0072 100644 --- a/meilisearch-http/src/routes/stop_words.rs +++ b/meilisearch-http/src/routes/stop_words.rs @@ -1,6 +1,9 @@ +use std::collections::BTreeSet; + use http::StatusCode; use tide::response::IntoResponse; use tide::{Context, Response}; +use meilisearch_core::settings::{SettingsUpdate, UpdateState}; use crate::error::{ResponseError, SResult}; use crate::helpers::tide::ContextExt; @@ -33,18 +36,17 @@ pub async fn update(mut ctx: Context) -> SResult { ctx.is_allowed(SettingsRead)?; let index = ctx.index()?; - let data: Vec = ctx.body_json().await.map_err(ResponseError::bad_request)?; + let data: BTreeSet = ctx.body_json().await.map_err(ResponseError::bad_request)?; let db = &ctx.state().db; let mut writer = db.update_write_txn().map_err(ResponseError::internal)?; - let mut stop_words_update = index.stop_words_update(); - for stop_word in data { - stop_words_update.add_stop_word(stop_word); - } + let settings = SettingsUpdate { + stop_words: UpdateState::Update(data), + .. SettingsUpdate::default() + }; - let update_id = stop_words_update - .finalize(&mut writer) + let update_id = index.settings_update(&mut writer, settings) .map_err(ResponseError::internal)?; writer.commit().map_err(ResponseError::internal)?; @@ -62,10 +64,12 @@ pub async fn delete(ctx: Context) -> SResult { let db = &ctx.state().db; let mut writer = db.update_write_txn().map_err(ResponseError::internal)?; - let stop_words_deletion = index.stop_words_update(); + let settings = SettingsUpdate { + stop_words: UpdateState::Clear, + .. SettingsUpdate::default() + }; - let update_id = stop_words_deletion - .finalize(&mut writer) + let update_id = index.settings_update(&mut writer, settings) .map_err(ResponseError::internal)?; writer.commit().map_err(ResponseError::internal)?; diff --git a/meilisearch-http/src/routes/synonym.rs b/meilisearch-http/src/routes/synonym.rs index 15d7e7ba4..a8aef2685 100644 --- a/meilisearch-http/src/routes/synonym.rs +++ b/meilisearch-http/src/routes/synonym.rs @@ -1,9 +1,10 @@ -use std::collections::HashMap; +use std::collections::BTreeMap; use http::StatusCode; use tide::response::IntoResponse; use tide::{Context, Response}; use indexmap::IndexMap; +use meilisearch_core::settings::{SettingsUpdate, UpdateState}; use crate::error::{ResponseError, SResult}; use crate::helpers::tide::ContextExt; @@ -47,21 +48,19 @@ pub async fn get(ctx: Context) -> SResult { pub async fn update(mut ctx: Context) -> SResult { ctx.is_allowed(SettingsWrite)?; - let data: HashMap> = ctx.body_json().await.map_err(ResponseError::bad_request)?; + let data: BTreeMap> = ctx.body_json().await.map_err(ResponseError::bad_request)?; let index = ctx.index()?; let db = &ctx.state().db; let mut writer = db.update_write_txn().map_err(ResponseError::internal)?; - let mut synonyms_update = index.synonyms_update(); + let settings = SettingsUpdate { + synonyms: UpdateState::Update(data), + .. SettingsUpdate::default() + }; - for (input, synonyms) in data { - synonyms_update.add_synonym(input, synonyms.into_iter()); - } - - let update_id = synonyms_update - .finalize(&mut writer) + let update_id = index.settings_update(&mut writer, settings) .map_err(ResponseError::internal)?; writer.commit().map_err(ResponseError::internal)?; @@ -86,10 +85,7 @@ pub async fn delete(ctx: Context) -> SResult { .. SettingsUpdate::default() }; - let synonyms_update = index.synonyms_update(); - - let update_id = synonyms_update - .finalize(&mut writer) + let update_id = index.settings_update(&mut writer, settings) .map_err(ResponseError::internal)?; writer.commit().map_err(ResponseError::internal)?; diff --git a/meilisearch-schema/src/error.rs b/meilisearch-schema/src/error.rs new file mode 100644 index 000000000..e37f12fdb --- /dev/null +++ b/meilisearch-schema/src/error.rs @@ -0,0 +1,20 @@ + +use std::{error, fmt}; + +pub type SResult = Result; + +#[derive(Debug)] +pub enum Error { + MaxFieldsLimitExceeded, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use self::Error::*; + match self { + MaxFieldsLimitExceeded => write!(f, "The maximum of possible reatributed field id has been reached"), + } + } +} + +impl error::Error for Error {} diff --git a/meilisearch-schema/src/fields_map.rs b/meilisearch-schema/src/fields_map.rs new file mode 100644 index 000000000..642eea9b9 --- /dev/null +++ b/meilisearch-schema/src/fields_map.rs @@ -0,0 +1,91 @@ +use std::io::{Read, Write}; +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::{SResult, SchemaAttr}; + +pub type FieldId = SchemaAttr; + +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FieldsMap { + name_map: HashMap, + id_map: HashMap, + next_id: FieldId +} + +impl FieldsMap { + pub fn len(&self) -> usize { + self.name_map.len() + } + + pub fn is_empty(&self) -> bool { + self.name_map.is_empty() + } + + pub fn insert>(&mut self, name: S) -> SResult { + let name = name.into(); + if let Some(id) = self.name_map.get(&name) { + return Ok(*id) + } + let id = self.next_id.into(); + self.next_id = self.next_id.next()?; + self.name_map.insert(name.clone(), id); + self.id_map.insert(id, name); + Ok(id) + } + + pub fn remove>(&mut self, name: S) { + let name = name.into(); + if let Some(id) = self.name_map.get(&name) { + self.id_map.remove(&id); + } + self.name_map.remove(&name); + } + + pub fn get_id>(&self, name: S) -> Option<&FieldId> { + let name = name.into(); + self.name_map.get(&name) + } + + pub fn get_name>(&self, id: I) -> Option<&String> { + self.id_map.get(&id.into()) + } + + pub fn read_from_bin(reader: R) -> bincode::Result { + bincode::deserialize_from(reader) + } + + pub fn write_to_bin(&self, writer: W) -> bincode::Result<()> { + bincode::serialize_into(writer, &self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fields_map() { + let mut fields_map = FieldsMap::default(); + assert_eq!(fields_map.insert("id").unwrap(), 0.into()); + assert_eq!(fields_map.insert("title").unwrap(), 1.into()); + assert_eq!(fields_map.insert("descritpion").unwrap(), 2.into()); + assert_eq!(fields_map.insert("id").unwrap(), 0.into()); + assert_eq!(fields_map.insert("title").unwrap(), 1.into()); + assert_eq!(fields_map.insert("descritpion").unwrap(), 2.into()); + assert_eq!(fields_map.get_id("id"), Some(&0.into())); + assert_eq!(fields_map.get_id("title"), Some(&1.into())); + assert_eq!(fields_map.get_id("descritpion"), Some(&2.into())); + assert_eq!(fields_map.get_id("date"), None); + assert_eq!(fields_map.len(), 3); + assert_eq!(fields_map.get_name(0), Some(&"id".to_owned())); + assert_eq!(fields_map.get_name(1), Some(&"title".to_owned())); + assert_eq!(fields_map.get_name(2), Some(&"descritpion".to_owned())); + assert_eq!(fields_map.get_name(4), None); + fields_map.remove("title"); + assert_eq!(fields_map.get_id("title"), None); + assert_eq!(fields_map.insert("title").unwrap(), 3.into()); + assert_eq!(fields_map.len(), 3); + } +} diff --git a/meilisearch-schema/src/lib.rs b/meilisearch-schema/src/lib.rs index f02ef2ac9..502b96828 100644 --- a/meilisearch-schema/src/lib.rs +++ b/meilisearch-schema/src/lib.rs @@ -1,239 +1,13 @@ -use std::collections::{BTreeMap, HashMap}; -use std::ops::BitOr; -use std::sync::Arc; -use std::{fmt, u16}; +mod error; +mod fields_map; +mod schema; -use indexmap::IndexMap; +pub use error::{Error, SResult}; +pub use fields_map::{FieldsMap, FieldId}; +pub use schema::{Schema, IndexedPos}; use serde::{Deserialize, Serialize}; -pub const DISPLAYED: SchemaProps = SchemaProps { - displayed: true, - indexed: false, - ranked: false, -}; -pub const INDEXED: SchemaProps = SchemaProps { - displayed: false, - indexed: true, - ranked: false, -}; -pub const RANKED: SchemaProps = SchemaProps { - displayed: false, - indexed: false, - ranked: true, -}; - -#[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SchemaProps { - #[serde(default)] - pub displayed: bool, - - #[serde(default)] - pub indexed: bool, - - #[serde(default)] - pub ranked: bool, -} - -impl SchemaProps { - pub fn is_displayed(self) -> bool { - self.displayed - } - - pub fn is_indexed(self) -> bool { - self.indexed - } - - pub fn is_ranked(self) -> bool { - self.ranked - } -} - -impl BitOr for SchemaProps { - type Output = Self; - - fn bitor(self, other: Self) -> Self::Output { - SchemaProps { - displayed: self.displayed | other.displayed, - indexed: self.indexed | other.indexed, - ranked: self.ranked | other.ranked, - } - } -} - -impl fmt::Debug for SchemaProps { - #[allow(non_camel_case_types)] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - #[derive(Debug)] - struct DISPLAYED; - - #[derive(Debug)] - struct INDEXED; - - #[derive(Debug)] - struct RANKED; - - let mut debug_set = f.debug_set(); - - if self.displayed { - debug_set.entry(&DISPLAYED); - } - - if self.indexed { - debug_set.entry(&INDEXED); - } - - if self.ranked { - debug_set.entry(&RANKED); - } - - debug_set.finish() - } -} - -#[derive(Serialize, Deserialize)] -pub struct SchemaBuilder { - identifier: String, - attributes: IndexMap, -} - -impl SchemaBuilder { - - pub fn with_identifier>(name: S) -> SchemaBuilder { - SchemaBuilder { - identifier: name.into(), - attributes: IndexMap::new(), - } - } - - pub fn new_attribute>(&mut self, name: S, props: SchemaProps) -> SchemaAttr { - let len = self.attributes.len(); - if self.attributes.insert(name.into(), props).is_some() { - panic!("Field already inserted.") - } - SchemaAttr(len as u16) - } - - pub fn build(self) -> Schema { - let mut attrs = HashMap::new(); - let mut props = Vec::new(); - - for (i, (name, prop)) in self.attributes.into_iter().enumerate() { - attrs.insert(name.clone(), SchemaAttr(i as u16)); - props.push((name, prop)); - } - - let identifier = self.identifier; - Schema { - inner: Arc::new(InnerSchema { - identifier, - attrs, - props, - }), - } - } -} - -#[derive(Clone, PartialEq, Eq)] -pub struct Schema { - inner: Arc, -} - -#[derive(Clone, PartialEq, Eq)] -struct InnerSchema { - identifier: String, - attrs: HashMap, - props: Vec<(String, SchemaProps)>, -} - -impl Schema { - pub fn to_builder(&self) -> SchemaBuilder { - let identifier = self.inner.identifier.clone(); - let attributes = self.attributes_ordered(); - SchemaBuilder { - identifier, - attributes, - } - } - - fn attributes_ordered(&self) -> IndexMap { - let mut ordered = BTreeMap::new(); - for (name, attr) in &self.inner.attrs { - let (_, props) = self.inner.props[attr.0 as usize]; - ordered.insert(attr.0, (name, props)); - } - - let mut attributes = IndexMap::with_capacity(ordered.len()); - for (_, (name, props)) in ordered { - attributes.insert(name.clone(), props); - } - - attributes - } - - pub fn number_of_attributes(&self) -> usize { - self.inner.attrs.len() - } - - pub fn props(&self, attr: SchemaAttr) -> SchemaProps { - let (_, props) = self.inner.props[attr.0 as usize]; - props - } - - pub fn identifier_name(&self) -> &str { - &self.inner.identifier - } - - pub fn attribute>(&self, name: S) -> Option { - self.inner.attrs.get(name.as_ref()).cloned() - } - - pub fn attribute_name(&self, attr: SchemaAttr) -> &str { - let (name, _) = &self.inner.props[attr.0 as usize]; - name - } - - pub fn into_iter<'a>(&'a self) -> impl Iterator + 'a { - self.inner.props.clone().into_iter() - } - - pub fn iter<'a>(&'a self) -> impl Iterator + 'a { - self.inner.props.iter().map(move |(name, prop)| { - let attr = self.inner.attrs.get(name).unwrap(); - (name.as_str(), *attr, *prop) - }) - } -} - -impl Serialize for Schema { - fn serialize(&self, serializer: S) -> Result - where - S: serde::ser::Serializer, - { - self.to_builder().serialize(serializer) - } -} - -impl<'de> Deserialize<'de> for Schema { - fn deserialize(deserializer: D) -> Result - where - D: serde::de::Deserializer<'de>, - { - let builder = SchemaBuilder::deserialize(deserializer)?; - Ok(builder.build()) - } -} - -impl fmt::Debug for Schema { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let builder = self.to_builder(); - f.debug_struct("Schema") - .field("identifier", &builder.identifier) - .field("attributes", &builder.attributes) - .finish() - } -} - -#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[derive(Serialize, Deserialize, Debug, Copy, Clone, Default, PartialOrd, Ord, PartialEq, Eq, Hash)] pub struct SchemaAttr(pub u16); impl SchemaAttr { @@ -249,346 +23,620 @@ impl SchemaAttr { SchemaAttr(u16::max_value()) } - pub fn next(self) -> Option { - self.0.checked_add(1).map(SchemaAttr) + pub fn next(self) -> SResult { + self.0.checked_add(1).map(SchemaAttr).ok_or(Error::MaxFieldsLimitExceeded) } - pub fn prev(self) -> Option { - self.0.checked_sub(1).map(SchemaAttr) + pub fn prev(self) -> SResult { + self.0.checked_sub(1).map(SchemaAttr).ok_or(Error::MaxFieldsLimitExceeded) } } -impl fmt::Display for SchemaAttr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) +impl From for SchemaAttr { + fn from(value: u16) -> SchemaAttr { + SchemaAttr(value) } } -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Diff { - IdentChange { - old: String, - new: String, - }, - AttrMove { - name: String, - old: usize, - new: usize, - }, - AttrPropsChange { - name: String, - old: SchemaProps, - new: SchemaProps, - }, - NewAttr { - name: String, - pos: usize, - props: SchemaProps, - }, - RemovedAttr { - name: String, - }, -} - -pub fn diff(old: &Schema, new: &Schema) -> Vec { - use Diff::{AttrMove, AttrPropsChange, IdentChange, NewAttr, RemovedAttr}; - - let mut differences = Vec::new(); - let old = old.to_builder(); - let new = new.to_builder(); - - // check if the old identifier differs from the new one - if old.identifier != new.identifier { - let old = old.identifier; - let new = new.identifier; - differences.push(IdentChange { old, new }); - } - - // compare all old attributes positions - // and properties with the new ones - for (pos, (name, props)) in old.attributes.iter().enumerate() { - match new.attributes.get_full(name) { - Some((npos, _, nprops)) => { - if pos != npos { - let name = name.clone(); - differences.push(AttrMove { - name, - old: pos, - new: npos, - }); - } - if props != nprops { - let name = name.clone(); - differences.push(AttrPropsChange { - name, - old: *props, - new: *nprops, - }); - } - } - None => differences.push(RemovedAttr { name: name.clone() }), - } - } - - // retrieve all attributes that - // were not present in the old schema - for (pos, (name, props)) in new.attributes.iter().enumerate() { - if !old.attributes.contains_key(name) { - let name = name.clone(); - differences.push(NewAttr { - name, - pos, - props: *props, - }); - } - } - - differences -} - - -// The diff_transposition return the transpotion matrix to apply during the documents rewrite process. -// e.g. -// old_schema: ["id", "title", "description", "tags", "date"] -// new_schema: ["title", "tags", "id", "new", "position","description"] -// diff_transposition: [Some(2), Some(0), Some(5), Some(1), None] -// -// - attribute 0 (id) become attribute 2 -// - attribute 1 (title) become attribute 0 -// - attribute 2 (description) become attribute 5 -// - attribute 3 (tags) become attribute 1 -// - attribute 4 (date) is deleted -pub fn diff_transposition(old: &Schema, new: &Schema) -> Vec> { - let old = old.to_builder(); - let new = new.to_builder(); - - let old_attributes: Vec<&str> = old.attributes.iter().map(|(key, _)| key.as_str()).collect(); - let new_attributes: Vec<&str> = new.attributes.iter().map(|(key, _)| key.as_str()).collect(); - - let mut transpotition = Vec::new(); - - for (_pos, attr) in old_attributes.iter().enumerate() { - if let Some(npos) = new_attributes[..].iter().position(|x| x == attr) { - transpotition.push(Some(npos as u16)); - } else { - transpotition.push(None); - } - } - - transpotition -} - -pub fn generate_schema(identifier: String, indexed: Vec, displayed: Vec, ranked: Vec) -> Schema { - let mut map = IndexMap::new(); - - for item in indexed.iter() { - map.entry(item).or_insert(SchemaProps::default()).indexed = true; - } - for item in ranked.iter() { - map.entry(item).or_insert(SchemaProps::default()).ranked = true; - } - for item in displayed.iter() { - map.entry(item).or_insert(SchemaProps::default()).displayed = true; - } - let id = identifier.clone(); - map.entry(&id).or_insert(SchemaProps::default()); - - let mut builder = SchemaBuilder::with_identifier(identifier); - - for (key, value) in map { - builder.new_attribute(key, value); - } - - builder.build() -} - -#[cfg(test)] -mod tests { - use super::*; - use std::error::Error; - - #[test] - fn difference() { - use Diff::{AttrMove, AttrPropsChange, IdentChange, NewAttr, RemovedAttr}; - - let mut builder = SchemaBuilder::with_identifier("id"); - builder.new_attribute("alpha", DISPLAYED); - builder.new_attribute("beta", DISPLAYED | INDEXED); - builder.new_attribute("gamma", INDEXED); - builder.new_attribute("omega", INDEXED); - let old = builder.build(); - - let mut builder = SchemaBuilder::with_identifier("kiki"); - builder.new_attribute("beta", DISPLAYED | INDEXED); - builder.new_attribute("alpha", DISPLAYED | INDEXED); - builder.new_attribute("delta", RANKED); - builder.new_attribute("gamma", DISPLAYED); - let new = builder.build(); - - let differences = diff(&old, &new); - let expected = &[ - IdentChange { - old: format!("id"), - new: format!("kiki"), - }, - AttrMove { - name: format!("alpha"), - old: 0, - new: 1, - }, - AttrPropsChange { - name: format!("alpha"), - old: DISPLAYED, - new: DISPLAYED | INDEXED, - }, - AttrMove { - name: format!("beta"), - old: 1, - new: 0, - }, - AttrMove { - name: format!("gamma"), - old: 2, - new: 3, - }, - AttrPropsChange { - name: format!("gamma"), - old: INDEXED, - new: DISPLAYED, - }, - RemovedAttr { - name: format!("omega"), - }, - NewAttr { - name: format!("delta"), - pos: 2, - props: RANKED, - }, - ]; - - assert_eq!(&differences, expected) - } - - #[test] - fn serialize_deserialize() -> bincode::Result<()> { - let mut builder = SchemaBuilder::with_identifier("id"); - builder.new_attribute("alpha", DISPLAYED); - builder.new_attribute("beta", DISPLAYED | INDEXED); - builder.new_attribute("gamma", INDEXED); - let schema = builder.build(); - - let mut buffer = Vec::new(); - bincode::serialize_into(&mut buffer, &schema)?; - let schema2 = bincode::deserialize_from(buffer.as_slice())?; - - assert_eq!(schema, schema2); - - Ok(()) - } - - #[test] - fn serialize_deserialize_toml() -> Result<(), Box> { - let mut builder = SchemaBuilder::with_identifier("id"); - builder.new_attribute("alpha", DISPLAYED); - builder.new_attribute("beta", DISPLAYED | INDEXED); - builder.new_attribute("gamma", INDEXED); - let schema = builder.build(); - - let buffer = toml::to_vec(&schema)?; - let schema2 = toml::from_slice(buffer.as_slice())?; - - assert_eq!(schema, schema2); - - let data = r#" - identifier = "id" - - [attributes."alpha"] - displayed = true - - [attributes."beta"] - displayed = true - indexed = true - - [attributes."gamma"] - indexed = true - "#; - let schema2 = toml::from_str(data)?; - assert_eq!(schema, schema2); - - Ok(()) - } - - #[test] - fn serialize_deserialize_json() -> Result<(), Box> { - let mut builder = SchemaBuilder::with_identifier("id"); - builder.new_attribute("alpha", DISPLAYED); - builder.new_attribute("beta", DISPLAYED | INDEXED); - builder.new_attribute("gamma", INDEXED); - let schema = builder.build(); - - let buffer = serde_json::to_vec(&schema)?; - let schema2 = serde_json::from_slice(buffer.as_slice())?; - - assert_eq!(schema, schema2); - - let data = r#" - { - "identifier": "id", - "attributes": { - "alpha": { - "displayed": true - }, - "beta": { - "displayed": true, - "indexed": true - }, - "gamma": { - "indexed": true - } - } - }"#; - let schema2 = serde_json::from_str(data)?; - assert_eq!(schema, schema2); - - Ok(()) - } - - #[test] - fn debug_output() { - use std::fmt::Write as _; - - let mut builder = SchemaBuilder::with_identifier("id"); - builder.new_attribute("alpha", DISPLAYED); - builder.new_attribute("beta", DISPLAYED | INDEXED); - builder.new_attribute("gamma", INDEXED); - let schema = builder.build(); - - let mut output = String::new(); - let _ = write!(&mut output, "{:#?}", schema); - - let expected = r#"Schema { - identifier: "id", - attributes: { - "alpha": { - DISPLAYED, - }, - "beta": { - DISPLAYED, - INDEXED, - }, - "gamma": { - INDEXED, - }, - }, -}"#; - - assert_eq!(output, expected); - - let mut output = String::new(); - let _ = write!(&mut output, "{:?}", schema); - - let expected = r#"Schema { identifier: "id", attributes: {"alpha": {DISPLAYED}, "beta": {DISPLAYED, INDEXED}, "gamma": {INDEXED}} }"#; - - assert_eq!(output, expected); +impl Into for SchemaAttr { + fn into(self) -> u16 { + self.0 } } + + + +// use std::collections::{BTreeMap, HashMap}; +// use std::ops::BitOr; +// use std::sync::Arc; +// use std::{fmt, u16}; + +// use indexmap::IndexMap; +// use serde::{Deserialize, Serialize}; + +// pub const DISPLAYED: SchemaProps = SchemaProps { +// displayed: true, +// indexed: false, +// ranked: false, +// }; +// pub const INDEXED: SchemaProps = SchemaProps { +// displayed: false, +// indexed: true, +// ranked: false, +// }; +// pub const RANKED: SchemaProps = SchemaProps { +// displayed: false, +// indexed: false, +// ranked: true, +// }; + +// #[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +// pub struct SchemaProps { +// #[serde(default)] +// pub displayed: bool, + +// #[serde(default)] +// pub indexed: bool, + +// #[serde(default)] +// pub ranked: bool, +// } + +// impl SchemaProps { +// pub fn is_displayed(self) -> bool { +// self.displayed +// } + +// pub fn is_indexed(self) -> bool { +// self.indexed +// } + +// pub fn is_ranked(self) -> bool { +// self.ranked +// } +// } + +// impl BitOr for SchemaProps { +// type Output = Self; + +// fn bitor(self, other: Self) -> Self::Output { +// SchemaProps { +// displayed: self.displayed | other.displayed, +// indexed: self.indexed | other.indexed, +// ranked: self.ranked | other.ranked, +// } +// } +// } + +// impl fmt::Debug for SchemaProps { +// #[allow(non_camel_case_types)] +// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// #[derive(Debug)] +// struct DISPLAYED; + +// #[derive(Debug)] +// struct INDEXED; + +// #[derive(Debug)] +// struct RANKED; + +// let mut debug_set = f.debug_set(); + +// if self.displayed { +// debug_set.entry(&DISPLAYED); +// } + +// if self.indexed { +// debug_set.entry(&INDEXED); +// } + +// if self.ranked { +// debug_set.entry(&RANKED); +// } + +// debug_set.finish() +// } +// } + +// #[derive(Serialize, Deserialize)] +// pub struct SchemaBuilder { +// identifier: String, +// attributes: IndexMap, +// } + +// impl SchemaBuilder { + +// pub fn with_identifier>(name: S) -> SchemaBuilder { +// SchemaBuilder { +// identifier: name.into(), +// attributes: IndexMap::new(), +// } +// } + +// pub fn new_attribute>(&mut self, name: S, props: SchemaProps) -> SchemaAttr { +// let len = self.attributes.len(); +// if self.attributes.insert(name.into(), props).is_some() { +// panic!("Field already inserted.") +// } +// SchemaAttr(len as u16) +// } + +// pub fn build(self) -> Schema { +// let mut attrs = HashMap::new(); +// let mut props = Vec::new(); + +// for (i, (name, prop)) in self.attributes.into_iter().enumerate() { +// attrs.insert(name.clone(), SchemaAttr(i as u16)); +// props.push((name, prop)); +// } + +// let identifier = self.identifier; +// Schema { +// inner: Arc::new(InnerSchema { +// identifier, +// attrs, +// props, +// }), +// } +// } +// } + +// #[derive(Clone, PartialEq, Eq)] +// pub struct Schema { +// inner: Arc, +// } + +// #[derive(Clone, PartialEq, Eq)] +// struct InnerSchema { +// identifier: (String, u16), +// attrs: HashMap, +// props: Vec<(String, SchemaProps)>, +// } + +// impl Schema { +// pub fn to_builder(&self) -> SchemaBuilder { +// let identifier = self.inner.identifier.clone(); +// let attributes = self.attributes_ordered(); +// SchemaBuilder { +// identifier, +// attributes, +// } +// } + +// fn attributes_ordered(&self) -> IndexMap { +// let mut ordered = BTreeMap::new(); +// for (name, attr) in &self.inner.attrs { +// let (_, props) = self.inner.props[attr.0 as usize]; +// ordered.insert(attr.0, (name, props)); +// } + +// let mut attributes = IndexMap::with_capacity(ordered.len()); +// for (_, (name, props)) in ordered { +// attributes.insert(name.clone(), props); +// } + +// attributes +// } + +// pub fn number_of_attributes(&self) -> usize { +// self.inner.attrs.len() +// } + +// pub fn props(&self, attr: SchemaAttr) -> SchemaProps { +// let (_, props) = self.inner.props[attr.0 as usize]; +// props +// } + +// pub fn identifier_name(&self) -> &str { +// &self.inner.identifier +// } + +// pub fn attribute>(&self, name: S) -> Option { +// self.inner.attrs.get(name.as_ref()).cloned() +// } + +// pub fn attribute_name(&self, attr: SchemaAttr) -> &str { +// let (name, _) = &self.inner.props[attr.0 as usize]; +// name +// } + +// pub fn into_iter<'a>(&'a self) -> impl Iterator + 'a { +// self.inner.props.clone().into_iter() +// } + +// pub fn iter<'a>(&'a self) -> impl Iterator + 'a { +// self.inner.props.iter().map(move |(name, prop)| { +// let attr = self.inner.attrs.get(name).unwrap(); +// (name.as_str(), *attr, *prop) +// }) +// } +// } + +// impl Serialize for Schema { +// fn serialize(&self, serializer: S) -> Result +// where +// S: serde::ser::Serializer, +// { +// self.to_builder().serialize(serializer) +// } +// } + +// impl<'de> Deserialize<'de> for Schema { +// fn deserialize(deserializer: D) -> Result +// where +// D: serde::de::Deserializer<'de>, +// { +// let builder = SchemaBuilder::deserialize(deserializer)?; +// Ok(builder.build()) +// } +// } + +// impl fmt::Debug for Schema { +// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// let builder = self.to_builder(); +// f.debug_struct("Schema") +// .field("identifier", &builder.identifier) +// .field("attributes", &builder.attributes) +// .finish() +// } +// } + +// #[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)] +// pub struct SchemaAttr(pub u16); + +// impl SchemaAttr { +// pub const fn new(value: u16) -> SchemaAttr { +// SchemaAttr(value) +// } + +// pub const fn min() -> SchemaAttr { +// SchemaAttr(u16::min_value()) +// } + +// pub const fn max() -> SchemaAttr { +// SchemaAttr(u16::max_value()) +// } + +// pub fn next(self) -> Option { +// self.0.checked_add(1).map(SchemaAttr) +// } + +// pub fn prev(self) -> Option { +// self.0.checked_sub(1).map(SchemaAttr) +// } +// } + +// impl fmt::Display for SchemaAttr { +// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { +// self.0.fmt(f) +// } +// } + +// #[derive(Debug, Clone, PartialEq, Eq)] +// pub enum Diff { +// IdentChange { +// old: String, +// new: String, +// }, +// AttrMove { +// name: String, +// old: usize, +// new: usize, +// }, +// AttrPropsChange { +// name: String, +// old: SchemaProps, +// new: SchemaProps, +// }, +// NewAttr { +// name: String, +// pos: usize, +// props: SchemaProps, +// }, +// RemovedAttr { +// name: String, +// }, +// } + +// pub fn diff(old: &Schema, new: &Schema) -> Vec { +// use Diff::{AttrMove, AttrPropsChange, IdentChange, NewAttr, RemovedAttr}; + +// let mut differences = Vec::new(); +// let old = old.to_builder(); +// let new = new.to_builder(); + +// // check if the old identifier differs from the new one +// if old.identifier != new.identifier { +// let old = old.identifier; +// let new = new.identifier; +// differences.push(IdentChange { old, new }); +// } + +// // compare all old attributes positions +// // and properties with the new ones +// for (pos, (name, props)) in old.attributes.iter().enumerate() { +// match new.attributes.get_full(name) { +// Some((npos, _, nprops)) => { +// if pos != npos { +// let name = name.clone(); +// differences.push(AttrMove { +// name, +// old: pos, +// new: npos, +// }); +// } +// if props != nprops { +// let name = name.clone(); +// differences.push(AttrPropsChange { +// name, +// old: *props, +// new: *nprops, +// }); +// } +// } +// None => differences.push(RemovedAttr { name: name.clone() }), +// } +// } + +// // retrieve all attributes that +// // were not present in the old schema +// for (pos, (name, props)) in new.attributes.iter().enumerate() { +// if !old.attributes.contains_key(name) { +// let name = name.clone(); +// differences.push(NewAttr { +// name, +// pos, +// props: *props, +// }); +// } +// } + +// differences +// } + + +// // The diff_transposition return the transpotion matrix to apply during the documents rewrite process. +// // e.g. +// // old_schema: ["id", "title", "description", "tags", "date"] +// // new_schema: ["title", "tags", "id", "new", "position","description"] +// // diff_transposition: [Some(2), Some(0), Some(5), Some(1), None] +// // +// // - attribute 0 (id) become attribute 2 +// // - attribute 1 (title) become attribute 0 +// // - attribute 2 (description) become attribute 5 +// // - attribute 3 (tags) become attribute 1 +// // - attribute 4 (date) is deleted +// pub fn diff_transposition(old: &Schema, new: &Schema) -> Vec> { +// let old = old.to_builder(); +// let new = new.to_builder(); + +// let old_attributes: Vec<&str> = old.attributes.iter().map(|(key, _)| key.as_str()).collect(); +// let new_attributes: Vec<&str> = new.attributes.iter().map(|(key, _)| key.as_str()).collect(); + +// let mut transpotition = Vec::new(); + +// for (_pos, attr) in old_attributes.iter().enumerate() { +// if let Some(npos) = new_attributes[..].iter().position(|x| x == attr) { +// transpotition.push(Some(npos as u16)); +// } else { +// transpotition.push(None); +// } +// } + +// transpotition +// } + +// pub fn generate_schema(identifier: String, indexed: Vec, displayed: Vec, ranked: Vec) -> Schema { +// let mut map = IndexMap::new(); + +// for item in indexed.iter() { +// map.entry(item).or_insert(SchemaProps::default()).indexed = true; +// } +// for item in ranked.iter() { +// map.entry(item).or_insert(SchemaProps::default()).ranked = true; +// } +// for item in displayed.iter() { +// map.entry(item).or_insert(SchemaProps::default()).displayed = true; +// } +// let id = identifier.clone(); +// map.entry(&id).or_insert(SchemaProps::default()); + +// let mut builder = SchemaBuilder::with_identifier(identifier); + +// for (key, value) in map { +// builder.new_attribute(key, value); +// } + +// builder.build() +// } + +// #[cfg(test)] +// mod tests { +// use super::*; +// use std::error::Error; + +// #[test] +// fn difference() { +// use Diff::{AttrMove, AttrPropsChange, IdentChange, NewAttr, RemovedAttr}; + +// let mut builder = SchemaBuilder::with_identifier("id"); +// builder.new_attribute("alpha", DISPLAYED); +// builder.new_attribute("beta", DISPLAYED | INDEXED); +// builder.new_attribute("gamma", INDEXED); +// builder.new_attribute("omega", INDEXED); +// let old = builder.build(); + +// let mut builder = SchemaBuilder::with_identifier("kiki"); +// builder.new_attribute("beta", DISPLAYED | INDEXED); +// builder.new_attribute("alpha", DISPLAYED | INDEXED); +// builder.new_attribute("delta", RANKED); +// builder.new_attribute("gamma", DISPLAYED); +// let new = builder.build(); + +// let differences = diff(&old, &new); +// let expected = &[ +// IdentChange { +// old: format!("id"), +// new: format!("kiki"), +// }, +// AttrMove { +// name: format!("alpha"), +// old: 0, +// new: 1, +// }, +// AttrPropsChange { +// name: format!("alpha"), +// old: DISPLAYED, +// new: DISPLAYED | INDEXED, +// }, +// AttrMove { +// name: format!("beta"), +// old: 1, +// new: 0, +// }, +// AttrMove { +// name: format!("gamma"), +// old: 2, +// new: 3, +// }, +// AttrPropsChange { +// name: format!("gamma"), +// old: INDEXED, +// new: DISPLAYED, +// }, +// RemovedAttr { +// name: format!("omega"), +// }, +// NewAttr { +// name: format!("delta"), +// pos: 2, +// props: RANKED, +// }, +// ]; + +// assert_eq!(&differences, expected) +// } + +// #[test] +// fn serialize_deserialize() -> bincode::Result<()> { +// let mut builder = SchemaBuilder::with_identifier("id"); +// builder.new_attribute("alpha", DISPLAYED); +// builder.new_attribute("beta", DISPLAYED | INDEXED); +// builder.new_attribute("gamma", INDEXED); +// let schema = builder.build(); + +// let mut buffer = Vec::new(); +// bincode::serialize_into(&mut buffer, &schema)?; +// let schema2 = bincode::deserialize_from(buffer.as_slice())?; + +// assert_eq!(schema, schema2); + +// Ok(()) +// } + +// #[test] +// fn serialize_deserialize_toml() -> Result<(), Box> { +// let mut builder = SchemaBuilder::with_identifier("id"); +// builder.new_attribute("alpha", DISPLAYED); +// builder.new_attribute("beta", DISPLAYED | INDEXED); +// builder.new_attribute("gamma", INDEXED); +// let schema = builder.build(); + +// let buffer = toml::to_vec(&schema)?; +// let schema2 = toml::from_slice(buffer.as_slice())?; + +// assert_eq!(schema, schema2); + +// let data = r#" +// identifier = "id" + +// [attributes."alpha"] +// displayed = true + +// [attributes."beta"] +// displayed = true +// indexed = true + +// [attributes."gamma"] +// indexed = true +// "#; +// let schema2 = toml::from_str(data)?; +// assert_eq!(schema, schema2); + +// Ok(()) +// } + +// #[test] +// fn serialize_deserialize_json() -> Result<(), Box> { +// let mut builder = SchemaBuilder::with_identifier("id"); +// builder.new_attribute("alpha", DISPLAYED); +// builder.new_attribute("beta", DISPLAYED | INDEXED); +// builder.new_attribute("gamma", INDEXED); +// let schema = builder.build(); + +// let buffer = serde_json::to_vec(&schema)?; +// let schema2 = serde_json::from_slice(buffer.as_slice())?; + +// assert_eq!(schema, schema2); + +// let data = r#" +// { +// "identifier": "id", +// "attributes": { +// "alpha": { +// "displayed": true +// }, +// "beta": { +// "displayed": true, +// "indexed": true +// }, +// "gamma": { +// "indexed": true +// } +// } +// }"#; +// let schema2 = serde_json::from_str(data)?; +// assert_eq!(schema, schema2); + +// Ok(()) +// } + +// #[test] +// fn debug_output() { +// use std::fmt::Write as _; + +// let mut builder = SchemaBuilder::with_identifier("id"); +// builder.new_attribute("alpha", DISPLAYED); +// builder.new_attribute("beta", DISPLAYED | INDEXED); +// builder.new_attribute("gamma", INDEXED); +// let schema = builder.build(); + +// let mut output = String::new(); +// let _ = write!(&mut output, "{:#?}", schema); + +// let expected = r#"Schema { +// identifier: "id", +// attributes: { +// "alpha": { +// DISPLAYED, +// }, +// "beta": { +// DISPLAYED, +// INDEXED, +// }, +// "gamma": { +// INDEXED, +// }, +// }, +// }"#; + +// assert_eq!(output, expected); + +// let mut output = String::new(); +// let _ = write!(&mut output, "{:?}", schema); + +// let expected = r#"Schema { identifier: "id", attributes: {"alpha": {DISPLAYED}, "beta": {DISPLAYED, INDEXED}, "gamma": {INDEXED}} }"#; + +// assert_eq!(output, expected); +// } +// } diff --git a/meilisearch-schema/src/schema.rs b/meilisearch-schema/src/schema.rs new file mode 100644 index 000000000..a0c738cfa --- /dev/null +++ b/meilisearch-schema/src/schema.rs @@ -0,0 +1,141 @@ +use std::collections::{HashMap, HashSet}; + +use crate::{FieldsMap, FieldId, SResult, SchemaAttr}; + +pub type IndexedPos = SchemaAttr; + +#[derive(Default)] +pub struct Schema { + fields_map: FieldsMap, + + identifier: FieldId, + ranked: HashSet, + displayed: HashSet, + + indexed: Vec, + indexed_map: HashMap, +} + +impl Schema { + + pub fn with_identifier>(name: S) -> Schema { + let mut schema = Schema::default(); + let field_id = schema.fields_map.insert(name.into()).unwrap(); + schema.identifier = field_id; + + schema + } + + pub fn identifier(&self) -> String { + self.fields_map.get_name(self.identifier).unwrap().to_string() + } + + pub fn get_id>(&self, name: S) -> Option<&FieldId> { + self.fields_map.get_id(name) + } + + pub fn get_name>(&self, id: I) -> Option<&String> { + self.fields_map.get_name(id) + } + + pub fn contains>(&self, name: S) -> bool { + match self.fields_map.get_id(name.into()) { + Some(_) => true, + None => false, + } + } + + pub fn get_or_create_empty>(&mut self, name: S) -> SResult { + self.fields_map.insert(name) + } + + pub fn get_or_create + std::clone::Clone>(&mut self, name: S) -> SResult { + match self.fields_map.get_id(name.clone()) { + Some(id) => { + Ok(*id) + } + None => { + self.set_indexed(name.clone())?; + self.set_displayed(name) + } + } + } + + pub fn set_ranked>(&mut self, name: S) -> SResult { + let id = self.fields_map.insert(name.into())?; + self.ranked.insert(id); + Ok(id) + } + + pub fn set_displayed>(&mut self, name: S) -> SResult { + let id = self.fields_map.insert(name.into())?; + self.displayed.insert(id); + Ok(id) + } + + pub fn set_indexed>(&mut self, name: S) -> SResult<(FieldId, IndexedPos)> { + let id = self.fields_map.insert(name.into())?; + let pos = self.indexed.len() as u16; + self.indexed.push(id); + self.indexed_map.insert(id, pos.into()); + Ok((id, pos.into())) + } + + pub fn is_ranked>(&self, name: S) -> Option<&FieldId> { + match self.fields_map.get_id(name.into()) { + Some(id) => self.ranked.get(id), + None => None, + } + } + + pub fn is_displayed>(&self, name: S) -> Option<&FieldId> { + match self.fields_map.get_id(name.into()) { + Some(id) => self.displayed.get(id), + None => None, + } + } + + pub fn is_indexed>(&self, name: S) -> Option<&IndexedPos> { + match self.fields_map.get_id(name.into()) { + Some(id) => self.indexed_map.get(id), + None => None, + } + } + + pub fn id_is_ranked(&self, id: FieldId) -> bool { + self.ranked.get(&id).is_some() + } + + pub fn id_is_displayed(&self, id: FieldId) -> bool { + self.displayed.get(&id).is_some() + } + + pub fn id_is_indexed(&self, id: FieldId) -> Option<&IndexedPos> { + self.indexed_map.get(&id) + } + + pub fn update_ranked>(&mut self, data: impl IntoIterator) -> SResult<()> { + self.ranked = HashSet::new(); + for name in data { + self.set_ranked(name)?; + } + Ok(()) + } + + pub fn update_displayed>(&mut self, data: impl IntoIterator) -> SResult<()> { + self.displayed = HashSet::new(); + for name in data { + self.set_displayed(name)?; + } + Ok(()) + } + + pub fn update_indexed>(&mut self, data: Vec) -> SResult<()> { + self.indexed = Vec::new(); + self.indexed_map = HashMap::new(); + for name in data { + self.set_indexed(name)?; + } + Ok(()) + } +}