Merge pull request #691 from meilisearch/rewrite-indexer

Rewrite and simplify every indexer function

meilisearch-core/Cargo.toml

@@ -34,7 +34,7 @@ pest_derive = "2.0"
 regex = "1.3.6"
 sdset = "0.4.0"
 serde = { version = "1.0.105", features = ["derive"] }
-serde_json = "1.0.50"
+serde_json = { version = "1.0.50", features = ["preserve_order"] }
 siphasher = "0.3.2"
 slice-group-by = "0.2.6"
 unicase = "2.6.0"

meilisearch-core/benches/search_benchmark.rs

@@ -8,7 +8,7 @@ use std::fs::File;
 use std::io::BufReader;
 use std::iter;
 
-use meilisearch_core::Database;
+use meilisearch_core::{Database, DatabaseOptions};
 use meilisearch_core::{ProcessedUpdateResult, UpdateStatus};
 use meilisearch_core::settings::{Settings, SettingsUpdate};
 use meilisearch_schema::Schema;
@@ -17,7 +17,7 @@ use serde_json::Value;
 use criterion::{criterion_group, criterion_main, Criterion, BenchmarkId};
 
 fn prepare_database(path: &Path) -> Database {
-    let database = Database::open_or_create(path).unwrap();
+    let database = Database::open_or_create(path, DatabaseOptions::default()).unwrap();
     let db = &database;
 
     let (sender, receiver) = mpsc::sync_channel(100);

meilisearch-core/src/lib.rs

@@ -18,13 +18,13 @@ mod query_words_mapper;
 mod ranked_map;
 mod raw_document;
 mod reordered_attrs;
-mod update;
 pub mod criterion;
 pub mod facets;
 pub mod raw_indexer;
-pub mod settings;
 pub mod serde;
+pub mod settings;
 pub mod store;
+pub mod update;
 
 pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT};
 pub use self::error::{Error, HeedError, FstError, MResult, pest_error, FacetError};

meilisearch-core/src/serde/convert_to_number.rs

@@ -1,198 +0,0 @@
-use std::str::FromStr;
-
-use ordered_float::OrderedFloat;
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-use crate::Number;
-
-pub struct ConvertToNumber;
-
-impl ser::Serializer for ConvertToNumber {
-    type Ok = Number;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_char(self, _value: char) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "char" })
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(i64::from(value)))
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Signed(value))
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(u64::from(value)))
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Unsigned(value))
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(f64::from(value))))
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::Float(OrderedFloat(value)))
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(Number::from_str(value)?)
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnrankableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Err(SerializerError::UnrankableType { type_name: "map" })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct",
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnrankableType {
-            type_name: "struct variant",
-        })
-    }
-}

meilisearch-core/src/serde/convert_to_string.rs

@@ -1,279 +0,0 @@
-use serde::ser;
-use serde::Serialize;
-
-use super::SerializerError;
-
-pub struct ConvertToString;
-
-impl ser::Serializer for ConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-    type SerializeSeq = SeqConvertToString;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapConvertToString;
-    type SerializeStruct = StructConvertToString;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_str(self, value: &str) -> Result<Self::Ok, Self::Error> {
-        Ok(value.to_string())
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Ok(String::new())
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Ok(SeqConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructConvertToString {
-            text: String::new(),
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapConvertToString {
-    text: String,
-}
-
-impl ser::SerializeMap for MapConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        self.text.push_str(" ");
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}
-
-pub struct StructConvertToString {
-    text: String,
-}
-
-impl ser::SerializeStruct for StructConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let value = value.serialize(ConvertToString)?;
-        self.text.push_str(key);
-        self.text.push_str(" ");
-        self.text.push_str(&value);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}
-
-pub struct SeqConvertToString {
-    text: String,
-}
-
-impl ser::SerializeSeq for SeqConvertToString {
-    type Ok = String;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.text.push_str(&text);
-        self.text.push_str(" ");
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.text)
-    }
-}

meilisearch-core/src/serde/extract_document_id.rs

@@ -1,310 +0,0 @@
-use std::hash::{Hash, Hasher};
-
-use crate::DocumentId;
-use serde::{ser, Serialize};
-use serde_json::{Value, Number};
-use siphasher::sip::SipHasher;
-
-use super::{ConvertToString, SerializerError};
-
-pub fn extract_document_id<D>(
-    primary_key: &str,
-    document: &D,
-) -> Result<Option<DocumentId>, SerializerError>
-where
-    D: serde::Serialize,
-{
-    let serializer = ExtractDocumentId { primary_key };
-    document.serialize(serializer)
-}
-
-fn validate_number(value: &Number) -> Option<String> {
-    if value.is_f64() {
-        return None
-    }
-    Some(value.to_string())
-}
-
-fn validate_string(value: &str) -> Option<String> {
-    if value.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') {
-        Some(value.to_string())
-    } else {
-        None
-    }
-}
-
-pub fn value_to_string(value: &Value) -> Option<String> {
-    match value {
-        Value::Null => None,
-        Value::Bool(_) => None,
-        Value::Number(value) => validate_number(value),
-        Value::String(value) => validate_string(value),
-        Value::Array(_) => None,
-        Value::Object(_) => None,
-    }
-}
-
-pub fn compute_document_id<H: Hash>(t: H) -> DocumentId {
-    let mut s = SipHasher::new();
-    t.hash(&mut s);
-    let hash = s.finish();
-    DocumentId(hash)
-}
-
-struct ExtractDocumentId<'a> {
-    primary_key: &'a str,
-}
-
-impl<'a> ser::Serializer for ExtractDocumentId<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = ExtractDocumentIdMapSerializer<'a>;
-    type SerializeStruct = ExtractDocumentIdStructSerializer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _value: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _value: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let serializer = ExtractDocumentIdMapSerializer {
-            primary_key: self.primary_key,
-            document_id: None,
-            current_key_name: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let serializer = ExtractDocumentIdStructSerializer {
-            primary_key: self.primary_key,
-            document_id: None,
-        };
-
-        Ok(serializer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct ExtractDocumentIdMapSerializer<'a> {
-    primary_key: &'a str,
-    document_id: Option<DocumentId>,
-    current_key_name: Option<String>,
-}
-
-impl<'a> ser::SerializeMap for ExtractDocumentIdMapSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: Serialize,
-        V: Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-
-        if self.primary_key == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(|s| compute_document_id(&s)) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}
-
-pub struct ExtractDocumentIdStructSerializer<'a> {
-    primary_key: &'a str,
-    document_id: Option<DocumentId>,
-}
-
-impl<'a> ser::SerializeStruct for ExtractDocumentIdStructSerializer<'a> {
-    type Ok = Option<DocumentId>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        if self.primary_key == key {
-            let value = serde_json::to_string(value).and_then(|s| serde_json::from_str(&s))?;
-            match value_to_string(&value).map(compute_document_id) {
-                Some(document_id) => self.document_id = Some(document_id),
-                None => return Err(SerializerError::InvalidDocumentIdType),
-            }
-        }
-
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.document_id)
-    }
-}

meilisearch-core/src/serde/indexer.rs

@@ -1,362 +0,0 @@
-use meilisearch_schema::IndexedPos;
-use serde::ser;
-use serde::Serialize;
-
-use super::{ConvertToString, SerializerError};
-use crate::raw_indexer::RawIndexer;
-use crate::DocumentId;
-
-pub struct Indexer<'a> {
-    pub pos: IndexedPos,
-    pub indexer: &'a mut RawIndexer,
-    pub document_id: DocumentId,
-}
-
-impl<'a> ser::Serializer for Indexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-    type SerializeSeq = SeqIndexer<'a>;
-    type SerializeTuple = TupleIndexer<'a>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapIndexer<'a>;
-    type SerializeStruct = StructIndexer<'a>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    fn serialize_bool(self, _value: bool) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_char(self, value: char) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i8(self, value: i8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i16(self, value: i16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i32(self, value: i32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_i64(self, value: i64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u8(self, value: u8) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u16(self, value: u16) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u32(self, value: u32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_u64(self, value: u64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f32(self, value: f32) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_f64(self, value: f64) -> Result<Self::Ok, Self::Error> {
-        let text = value.serialize(ConvertToString)?;
-        self.serialize_str(&text)
-    }
-
-    fn serialize_str(self, text: &str) -> Result<Self::Ok, Self::Error> {
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.pos, text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnindexableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_some<T: ?Sized>(self, value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        let number_of_words = self
-            .indexer
-            .index_text(self.document_id, self.pos, &text);
-        Ok(Some(number_of_words))
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Ok(None)
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnindexableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        let indexer = SeqIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        let indexer = TupleIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        let indexer = MapIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        let indexer = StructIndexer {
-            pos: self.pos,
-            document_id: self.document_id,
-            indexer: self.indexer,
-            texts: Vec::new(),
-        };
-
-        Ok(indexer)
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnindexableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct SeqIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeSeq for SeqIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}
-
-pub struct MapIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeMap for MapIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = key.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}
-
-pub struct StructIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeStruct for StructIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key_text = key.to_owned();
-        let value_text = value.serialize(ConvertToString)?;
-        self.texts.push(key_text);
-        self.texts.push(value_text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}
-
-pub struct TupleIndexer<'a> {
-    pos: IndexedPos,
-    document_id: DocumentId,
-    indexer: &'a mut RawIndexer,
-    texts: Vec<String>,
-}
-
-impl<'a> ser::SerializeTuple for TupleIndexer<'a> {
-    type Ok = Option<usize>;
-    type Error = SerializerError;
-
-    fn serialize_element<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: Serialize,
-    {
-        let text = value.serialize(ConvertToString)?;
-        self.texts.push(text);
-        Ok(())
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        let texts = self.texts.iter().map(String::as_str);
-        self.indexer
-            .index_text_seq(self.document_id, self.pos, texts);
-        Ok(None)
-    }
-}

meilisearch-core/src/serde/mod.rs

@@ -1,26 +1,6 @@
-macro_rules! forward_to_unserializable_type {
-    ($($ty:ident => $se_method:ident,)*) => {
-        $(
-            fn $se_method(self, _v: $ty) -> Result<Self::Ok, Self::Error> {
-                Err(SerializerError::UnserializableType { type_name: "$ty" })
-            }
-        )*
-    }
-}
-
-mod convert_to_number;
-mod convert_to_string;
 mod deserializer;
-mod extract_document_id;
-mod indexer;
-mod serializer;
 
-pub use self::convert_to_number::ConvertToNumber;
-pub use self::convert_to_string::ConvertToString;
 pub use self::deserializer::{Deserializer, DeserializerError};
-pub use self::extract_document_id::{compute_document_id, extract_document_id, value_to_string};
-pub use self::indexer::Indexer;
-pub use self::serializer::{serialize_value, serialize_value_with_id, Serializer};
 
 use std::{error::Error, fmt};
 
@@ -33,7 +13,7 @@ use crate::ParseNumberError;
 #[derive(Debug)]
 pub enum SerializerError {
     DocumentIdNotFound,
-    InvalidDocumentIdType,
+    InvalidDocumentIdFormat,
     Zlmdb(heed::Error),
     SerdeJson(SerdeJsonError),
     ParseNumber(ParseNumberError),
@@ -56,7 +36,7 @@ impl fmt::Display for SerializerError {
             SerializerError::DocumentIdNotFound => {
                 f.write_str("serialized document does not have an id according to the schema")
             }
-            SerializerError::InvalidDocumentIdType => {
+            SerializerError::InvalidDocumentIdFormat => {
                 f.write_str("a document primary key can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")
             }
             SerializerError::Zlmdb(e) => write!(f, "heed related error: {}", e),

meilisearch-core/src/serde/serializer.rs

@@ -1,361 +0,0 @@
-use meilisearch_schema::{Schema, FieldId};
-use serde::ser;
-
-use crate::database::MainT;
-use crate::raw_indexer::RawIndexer;
-use crate::store::{DocumentsFields, DocumentsFieldsCounts};
-use crate::{DocumentId, RankedMap};
-
-use super::{ConvertToNumber, ConvertToString, Indexer, SerializerError};
-
-pub struct Serializer<'a, 'b> {
-    pub txn: &'a mut heed::RwTxn<'b, MainT>,
-    pub schema: &'a mut Schema,
-    pub document_store: DocumentsFields,
-    pub document_fields_counts: DocumentsFieldsCounts,
-    pub indexer: &'a mut RawIndexer,
-    pub ranked_map: &'a mut RankedMap,
-    pub document_id: DocumentId,
-}
-
-impl<'a, 'b> ser::Serializer for Serializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-    type SerializeSeq = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTuple = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleStruct = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeTupleVariant = ser::Impossible<Self::Ok, Self::Error>;
-    type SerializeMap = MapSerializer<'a, 'b>;
-    type SerializeStruct = StructSerializer<'a, 'b>;
-    type SerializeStructVariant = ser::Impossible<Self::Ok, Self::Error>;
-
-    forward_to_unserializable_type! {
-        bool => serialize_bool,
-        char => serialize_char,
-
-        i8  => serialize_i8,
-        i16 => serialize_i16,
-        i32 => serialize_i32,
-        i64 => serialize_i64,
-
-        u8  => serialize_u8,
-        u16 => serialize_u16,
-        u32 => serialize_u32,
-        u64 => serialize_u64,
-
-        f32 => serialize_f32,
-        f64 => serialize_f64,
-    }
-
-    fn serialize_str(self, _v: &str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "str" })
-    }
-
-    fn serialize_bytes(self, _v: &[u8]) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "&[u8]" })
-    }
-
-    fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_some<T: ?Sized>(self, _value: &T) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "Option",
-        })
-    }
-
-    fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "()" })
-    }
-
-    fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit struct",
-        })
-    }
-
-    fn serialize_unit_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-    ) -> Result<Self::Ok, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "unit variant",
-        })
-    }
-
-    fn serialize_newtype_struct<T: ?Sized>(
-        self,
-        _name: &'static str,
-        value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        value.serialize(self)
-    }
-
-    fn serialize_newtype_variant<T: ?Sized>(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _value: &T,
-    ) -> Result<Self::Ok, Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        Err(SerializerError::UnserializableType {
-            type_name: "newtype variant",
-        })
-    }
-
-    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "sequence",
-        })
-    }
-
-    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, Self::Error> {
-        Err(SerializerError::UnserializableType { type_name: "tuple" })
-    }
-
-    fn serialize_tuple_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleStruct, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple struct",
-        })
-    }
-
-    fn serialize_tuple_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeTupleVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "tuple variant",
-        })
-    }
-
-    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
-        Ok(MapSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-            current_key_name: None,
-        })
-    }
-
-    fn serialize_struct(
-        self,
-        _name: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStruct, Self::Error> {
-        Ok(StructSerializer {
-            txn: self.txn,
-            schema: self.schema,
-            document_id: self.document_id,
-            document_store: self.document_store,
-            document_fields_counts: self.document_fields_counts,
-            indexer: self.indexer,
-            ranked_map: self.ranked_map,
-        })
-    }
-
-    fn serialize_struct_variant(
-        self,
-        _name: &'static str,
-        _variant_index: u32,
-        _variant: &'static str,
-        _len: usize,
-    ) -> Result<Self::SerializeStructVariant, Self::Error> {
-        Err(SerializerError::UnserializableType {
-            type_name: "struct variant",
-        })
-    }
-}
-
-pub struct MapSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a mut Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-    current_key_name: Option<String>,
-}
-
-impl<'a, 'b> ser::SerializeMap for MapSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_key<T: ?Sized>(&mut self, key: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        self.current_key_name = Some(key);
-        Ok(())
-    }
-
-    fn serialize_value<T: ?Sized>(&mut self, value: &T) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        let key = self.current_key_name.take().unwrap();
-        self.serialize_entry(&key, value)
-    }
-
-    fn serialize_entry<K: ?Sized, V: ?Sized>(
-        &mut self,
-        key: &K,
-        value: &V,
-    ) -> Result<(), Self::Error>
-    where
-        K: ser::Serialize,
-        V: ser::Serialize,
-    {
-        let key = key.serialize(ConvertToString)?;
-        serialize_value(
-            self.txn,
-            key.as_str(),
-            self.schema,
-            self.document_id,
-            self.document_store,
-            self.document_fields_counts,
-            self.indexer,
-            self.ranked_map,
-            value,
-        )
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub struct StructSerializer<'a, 'b> {
-    txn: &'a mut heed::RwTxn<'b, MainT>,
-    schema: &'a mut Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    document_fields_counts: DocumentsFieldsCounts,
-    indexer: &'a mut RawIndexer,
-    ranked_map: &'a mut RankedMap,
-}
-
-impl<'a, 'b> ser::SerializeStruct for StructSerializer<'a, 'b> {
-    type Ok = ();
-    type Error = SerializerError;
-
-    fn serialize_field<T: ?Sized>(
-        &mut self,
-        key: &'static str,
-        value: &T,
-    ) -> Result<(), Self::Error>
-    where
-        T: ser::Serialize,
-    {
-        serialize_value(
-            self.txn,
-            key,
-            self.schema,
-            self.document_id,
-            self.document_store,
-            self.document_fields_counts,
-            self.indexer,
-            self.ranked_map,
-            value,
-        )
-    }
-
-    fn end(self) -> Result<Self::Ok, Self::Error> {
-        Ok(())
-    }
-}
-
-pub fn serialize_value<'a, T: ?Sized>(
-    txn: &mut heed::RwTxn<MainT>,
-    attribute: &str,
-    schema: &'a mut Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    documents_fields_counts: DocumentsFieldsCounts,
-    indexer: &mut RawIndexer,
-    ranked_map: &mut RankedMap,
-    value: &T,
-) -> Result<(), SerializerError>
-where
-    T: ser::Serialize,
-{
-    let field_id = schema.insert_and_index(&attribute)?;
-    serialize_value_with_id(
-        txn,
-        field_id,
-        schema,
-        document_id,
-        document_store,
-        documents_fields_counts,
-        indexer,
-        ranked_map,
-        value,
-    )
-}
-
-pub fn serialize_value_with_id<'a, T: ?Sized>(
-    txn: &mut heed::RwTxn<MainT>,
-    field_id: FieldId,
-    schema: &'a Schema,
-    document_id: DocumentId,
-    document_store: DocumentsFields,
-    documents_fields_counts: DocumentsFieldsCounts,
-    indexer: &mut RawIndexer,
-    ranked_map: &mut RankedMap,
-    value: &T,
-) -> Result<(), SerializerError>
-where
-    T: ser::Serialize,
-{
-    let serialized = serde_json::to_vec(value)?;
-    document_store.put_document_field(txn, document_id, field_id, &serialized)?;
-
-    if let Some(indexed_pos) = schema.is_indexed(field_id) {
-        let indexer = Indexer {
-            pos: *indexed_pos,
-            indexer,
-            document_id,
-        };
-        if let Some(number_of_words) = value.serialize(indexer)? {
-            documents_fields_counts.put_document_field_count(
-                txn,
-                document_id,
-                *indexed_pos,
-                number_of_words as u16,
-            )?;
-        }
-    }
-
-    if schema.is_ranked(field_id) {
-        let number = value.serialize(ConvertToNumber).unwrap_or_default();
-        ranked_map.insert(document_id, field_id, number);
-    }
-
-    Ok(())
-}

meilisearch-core/src/update/documents_addition.rs

@@ -2,15 +2,19 @@ use std::collections::HashMap;
 
 use fst::{set::OpBuilder, SetBuilder};
 use indexmap::IndexMap;
+use meilisearch_schema::{Schema, FieldId};
+use meilisearch_types::DocumentId;
 use sdset::{duo::Union, SetOperation};
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
+use serde_json::Value;
 
 use crate::database::{MainT, UpdateT};
 use crate::database::{UpdateEvent, UpdateEventsEmitter};
 use crate::facets;
 use crate::raw_indexer::RawIndexer;
-use crate::serde::{extract_document_id, serialize_value_with_id, Deserializer, Serializer};
-use crate::store;
+use crate::serde::Deserializer;
+use crate::store::{self, DocumentsFields, DocumentsFieldsCounts};
+use crate::update::helpers::{index_value, value_to_number, extract_document_id};
 use crate::update::{apply_documents_deletion, compute_short_prefixes, next_update_id, Update};
 use crate::{Error, MResult, RankedMap};
 
@@ -104,10 +108,45 @@ pub fn push_documents_addition<D: serde::Serialize>(
     Ok(last_update_id)
 }
 
+fn index_document(
+    writer: &mut heed::RwTxn<MainT>,
+    documents_fields: DocumentsFields,
+    documents_fields_counts: DocumentsFieldsCounts,
+    ranked_map: &mut RankedMap,
+    indexer: &mut RawIndexer,
+    schema: &Schema,
+    field_id: FieldId,
+    document_id: DocumentId,
+    value: &Value,
+) -> MResult<()>
+{
+    let serialized = serde_json::to_vec(value)?;
+    documents_fields.put_document_field(writer, document_id, field_id, &serialized)?;
+
+    if let Some(indexed_pos) = schema.is_indexed(field_id) {
+        let number_of_words = index_value(indexer, document_id, *indexed_pos, value);
+        if let Some(number_of_words) = number_of_words {
+            documents_fields_counts.put_document_field_count(
+                writer,
+                document_id,
+                *indexed_pos,
+                number_of_words as u16,
+            )?;
+        }
+    }
+
+    if schema.is_ranked(field_id) {
+        let number = value_to_number(value).unwrap_or_default();
+        ranked_map.insert(document_id, field_id, number);
+    }
+
+    Ok(())
+}
+
 pub fn apply_addition<'a, 'b>(
     writer: &'a mut heed::RwTxn<'b, MainT>,
     index: &store::Index,
-    addition: Vec<IndexMap<String, serde_json::Value>>,
+    new_documents: Vec<IndexMap<String, Value>>,
     partial: bool
 ) -> MResult<()> {
     let mut documents_additions = HashMap::new();
@@ -120,11 +159,8 @@ pub fn apply_addition<'a, 'b>(
     let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
 
     // 1. store documents ids for future deletion
-    for mut document in addition {
-        let document_id = match extract_document_id(&primary_key, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
+    for mut document in new_documents {
+        let document_id = extract_document_id(&primary_key, &document)?;
 
         if partial {
             let mut deserializer = Deserializer {
@@ -135,10 +171,8 @@ pub fn apply_addition<'a, 'b>(
                 fields: None,
             };
 
-            // retrieve the old document and
-            // update the new one with missing keys found in the old one
-            let result = Option::<HashMap<String, serde_json::Value>>::deserialize(&mut deserializer)?;
-            if let Some(old_document) = result {
+            let old_document = Option::<HashMap<String, Value>>::deserialize(&mut deserializer)?;
+            if let Some(old_document) = old_document {
                 for (key, value) in old_document {
                     document.entry(key).or_insert(value);
                 }
@@ -170,18 +204,23 @@ pub fn apply_addition<'a, 'b>(
 
     let mut indexer = RawIndexer::new(stop_words);
 
+    // For each document in this update
     for (document_id, document) in documents_additions {
-        let serializer = Serializer {
-            txn: writer,
-            schema: &mut schema,
-            document_store: index.documents_fields,
-            document_fields_counts: index.documents_fields_counts,
-            indexer: &mut indexer,
-            ranked_map: &mut ranked_map,
+        // For each key-value pair in the document.
+        for (attribute, value) in document {
+            let field_id = schema.insert_and_index(&attribute)?;
+            index_document(
+                writer,
+                index.documents_fields,
+                index.documents_fields_counts,
+                &mut ranked_map,
+                &mut indexer,
+                &schema,
+                field_id,
                 document_id,
-        };
-
-        document.serialize(serializer)?;
+                &value,
+            )?;
+        }
     }
 
     write_documents_addition_index(
@@ -200,17 +239,17 @@ pub fn apply_addition<'a, 'b>(
 pub fn apply_documents_partial_addition<'a, 'b>(
     writer: &'a mut heed::RwTxn<'b, MainT>,
     index: &store::Index,
-    addition: Vec<IndexMap<String, serde_json::Value>>,
+    new_documents: Vec<IndexMap<String, Value>>,
 ) -> MResult<()> {
-    apply_addition(writer, index, addition, true)
+    apply_addition(writer, index, new_documents, true)
 }
 
 pub fn apply_documents_addition<'a, 'b>(
     writer: &'a mut heed::RwTxn<'b, MainT>,
     index: &store::Index,
-    addition: Vec<IndexMap<String, serde_json::Value>>,
+    new_documents: Vec<IndexMap<String, Value>>,
 ) -> MResult<()> {
-    apply_addition(writer, index, addition, false)
+    apply_addition(writer, index, new_documents, false)
 }
 
 pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Index) -> MResult<()> {
@@ -253,21 +292,22 @@ pub fn reindex_all_documents(writer: &mut heed::RwTxn<MainT>, index: &store::Ind
     for document_id in documents_ids_to_reindex {
         for result in index.documents_fields.document_fields(writer, document_id)? {
             let (field_id, bytes) = result?;
-            let value: serde_json::Value = serde_json::from_slice(bytes)?;
+            let value: Value = serde_json::from_slice(bytes)?;
             ram_store.insert((document_id, field_id), value);
         }
 
-        for ((docid, field_id), value) in ram_store.drain() {
-            serialize_value_with_id(
+        // For each key-value pair in the document.
+        for ((document_id, field_id), value) in ram_store.drain() {
+            index_document(
                 writer,
-                field_id,
-                &schema,
-                docid,
                 index.documents_fields,
                 index.documents_fields_counts,
-                &mut indexer,
                 &mut ranked_map,
-                &value
+                &mut indexer,
+                &schema,
+                field_id,
+                document_id,
+                &value,
             )?;
         }
     }

meilisearch-core/src/update/documents_deletion.rs

@@ -1,13 +1,11 @@
 use std::collections::{BTreeSet, HashMap, HashSet};
 
 use fst::{SetBuilder, Streamer};
-use meilisearch_schema::Schema;
 use sdset::{duo::DifferenceByKey, SetBuf, SetOperation};
 
 use crate::database::{MainT, UpdateT};
 use crate::database::{UpdateEvent, UpdateEventsEmitter};
 use crate::facets;
-use crate::serde::extract_document_id;
 use crate::store;
 use crate::update::{next_update_id, compute_short_prefixes, Update};
 use crate::{DocumentId, Error, MResult, RankedMap};
@@ -37,21 +35,6 @@ impl DocumentsDeletion {
         self.documents.push(document_id);
     }
 
-    pub fn delete_document<D>(&mut self, schema: &Schema, document: D) -> MResult<()>
-    where
-        D: serde::Serialize,
-    {
-        let primary_key = schema.primary_key().ok_or(Error::MissingPrimaryKey)?;
-        let document_id = match extract_document_id(&primary_key, &document)? {
-            Some(id) => id,
-            None => return Err(Error::MissingDocumentId),
-        };
-
-        self.delete_document_by_id(document_id);
-
-        Ok(())
-    }
-
     pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
         let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
         let update_id = push_documents_deletion(

meilisearch-core/src/update/helpers.rs

@@ -0,0 +1,124 @@
+use std::fmt::Write as _;
+use std::hash::{Hash, Hasher};
+
+use indexmap::IndexMap;
+use meilisearch_schema::IndexedPos;
+use meilisearch_types::DocumentId;
+use ordered_float::OrderedFloat;
+use serde_json::Value;
+use siphasher::sip::SipHasher;
+
+use crate::raw_indexer::RawIndexer;
+use crate::serde::SerializerError;
+use crate::Number;
+
+/// Returns the number of words indexed or `None` if the type is unindexable.
+pub fn index_value(
+    indexer: &mut RawIndexer,
+    document_id: DocumentId,
+    indexed_pos: IndexedPos,
+    value: &Value,
+) -> Option<usize>
+{
+    match value {
+        Value::Null => None,
+        Value::Bool(boolean) => {
+            let text = boolean.to_string();
+            let number_of_words = indexer.index_text(document_id, indexed_pos, &text);
+            Some(number_of_words)
+        },
+        Value::Number(number) => {
+            let text = number.to_string();
+            Some(indexer.index_text(document_id, indexed_pos, &text))
+        },
+        Value::String(string) => {
+            Some(indexer.index_text(document_id, indexed_pos, &string))
+        },
+        Value::Array(_) => {
+            let text = value_to_string(value);
+            Some(indexer.index_text(document_id, indexed_pos, &text))
+        },
+        Value::Object(_) => {
+            let text = value_to_string(value);
+            Some(indexer.index_text(document_id, indexed_pos, &text))
+        },
+    }
+}
+
+/// Transforms the JSON Value type into a String.
+pub fn value_to_string(value: &Value) -> String {
+    fn internal_value_to_string(string: &mut String, value: &Value) {
+        match value {
+            Value::Null => (),
+            Value::Bool(boolean) => { let _ = write!(string, "{}", &boolean); },
+            Value::Number(number) => { let _ = write!(string, "{}", &number); },
+            Value::String(text) => string.push_str(&text),
+            Value::Array(array) => {
+                for value in array {
+                    internal_value_to_string(string, value);
+                    let _ = string.write_str(". ");
+                }
+            },
+            Value::Object(object) => {
+                for (key, value) in object {
+                    string.push_str(key);
+                    let _ = string.write_str(". ");
+                    internal_value_to_string(string, value);
+                    let _ = string.write_str(". ");
+                }
+            },
+        }
+    }
+
+    let mut string = String::new();
+    internal_value_to_string(&mut string, value);
+    string
+}
+
+/// Transforms the JSON Value type into a Number.
+pub fn value_to_number(value: &Value) -> Option<Number> {
+    use std::str::FromStr;
+
+    match value {
+        Value::Null => None,
+        Value::Bool(boolean) => Some(Number::Unsigned(*boolean as u64)),
+        Value::Number(number) => {
+            match (number.as_i64(), number.as_u64(), number.as_f64()) {
+                (Some(n), _, _) => Some(Number::Signed(n)),
+                (_, Some(n), _) => Some(Number::Unsigned(n)),
+                (_, _, Some(n)) => Some(Number::Float(OrderedFloat(n))),
+                (None, None, None) => None,
+            }
+        },
+        Value::String(string) => Number::from_str(string).ok(),
+        Value::Array(_array) => None,
+        Value::Object(_object) => None,
+    }
+}
+
+/// Validates a string representation to be a correct document id and
+/// returns the hash of the given type, this is the way we produce documents ids.
+pub fn compute_document_id(string: &str) -> Result<DocumentId, SerializerError> {
+    if string.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_') {
+        let mut s = SipHasher::new();
+        string.hash(&mut s);
+        Ok(DocumentId(s.finish()))
+    } else {
+        Err(SerializerError::InvalidDocumentIdFormat)
+    }
+}
+
+/// Extracts and validates the document id of a document.
+pub fn extract_document_id(primary_key: &str, document: &IndexMap<String, Value>) -> Result<DocumentId, SerializerError> {
+    match document.get(primary_key) {
+        Some(value) => {
+            let string = match value {
+                Value::Number(number) => number.to_string(),
+                Value::String(string) => string.clone(),
+                _ => return Err(SerializerError::InvalidDocumentIdFormat),
+            };
+            compute_document_id(&string)
+        }
+        None => Err(SerializerError::DocumentIdNotFound),
+    }
+}

meilisearch-core/src/update/mod.rs

@@ -3,13 +3,13 @@ mod customs_update;
 mod documents_addition;
 mod documents_deletion;
 mod settings_update;
+mod helpers;
 
 pub use self::clear_all::{apply_clear_all, push_clear_all};
 pub use self::customs_update::{apply_customs_update, push_customs_update};
-pub use self::documents_addition::{
-    apply_documents_addition, apply_documents_partial_addition, DocumentsAddition,
-};
+pub use self::documents_addition::{apply_documents_addition, apply_documents_partial_addition, DocumentsAddition};
 pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion};
+pub use self::helpers::{index_value, value_to_string, value_to_number, compute_document_id, extract_document_id};
 pub use self::settings_update::{apply_settings_update, push_settings_update};
 
 use std::cmp;
@@ -22,6 +22,7 @@ use indexmap::IndexMap;
 use log::debug;
 use sdset::Set;
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 
 use crate::{store, DocumentId, MResult};
 use crate::database::{MainT, UpdateT};
@@ -48,16 +49,16 @@ impl Update {
         }
     }
 
-    fn documents_addition(data: Vec<IndexMap<String, serde_json::Value>>) -> Update {
+    fn documents_addition(documents: Vec<IndexMap<String, Value>>) -> Update {
         Update {
-            data: UpdateData::DocumentsAddition(data),
+            data: UpdateData::DocumentsAddition(documents),
             enqueued_at: Utc::now(),
         }
     }
 
-    fn documents_partial(data: Vec<IndexMap<String, serde_json::Value>>) -> Update {
+    fn documents_partial(documents: Vec<IndexMap<String, Value>>) -> Update {
         Update {
-            data: UpdateData::DocumentsPartial(data),
+            data: UpdateData::DocumentsPartial(documents),
             enqueued_at: Utc::now(),
         }
     }
@@ -81,8 +82,8 @@ impl Update {
 pub enum UpdateData {
     ClearAll,
     Customs(Vec<u8>),
-    DocumentsAddition(Vec<IndexMap<String, serde_json::Value>>),
-    DocumentsPartial(Vec<IndexMap<String, serde_json::Value>>),
+    DocumentsAddition(Vec<IndexMap<String, Value>>),
+    DocumentsPartial(Vec<IndexMap<String, Value>>),
     DocumentsDeletion(Vec<DocumentId>),
     Settings(SettingsUpdate)
 }

meilisearch-http/src/routes/document.rs

@@ -3,6 +3,7 @@ use std::collections::{BTreeSet, HashSet};
 use actix_web::{web, HttpResponse};
 use actix_web_macros::{delete, get, post, put};
 use indexmap::IndexMap;
+use meilisearch_core::{update, Error};
 use serde::Deserialize;
 use serde_json::Value;
 
@@ -42,8 +43,7 @@ async fn get_document(
         .open_index(&path.index_uid)
         .ok_or(ResponseError::index_not_found(&path.index_uid))?;
 
-    let document_id = meilisearch_core::serde::compute_document_id(&path.document_id);
-
+    let document_id = update::compute_document_id(&path.document_id).map_err(Error::Serializer)?;
     let reader = data.db.main_read_txn()?;
 
     let response: Document = index
@@ -65,7 +65,8 @@ async fn delete_document(
         .db
         .open_index(&path.index_uid)
         .ok_or(ResponseError::index_not_found(&path.index_uid))?;
-    let document_id = meilisearch_core::serde::compute_document_id(&path.document_id);
+
+    let document_id = update::compute_document_id(&path.document_id).map_err(Error::Serializer)?;
 
     let mut update_writer = data.db.update_write_txn()?;
 
@@ -237,10 +238,9 @@ async fn delete_documents(
     let mut documents_deletion = index.documents_deletion();
 
     for document_id in body.into_inner() {
-        if let Some(document_id) = meilisearch_core::serde::value_to_string(&document_id) {
-            documents_deletion
-                .delete_document_by_id(meilisearch_core::serde::compute_document_id(document_id));
-        }
+        let document_id = update::value_to_string(&document_id);
+        let document_id = update::compute_document_id(&document_id).map_err(Error::Serializer)?;
+        documents_deletion.delete_document_by_id(document_id);
     }
 
     let update_id = documents_deletion.finalize(&mut writer)?;