Merge pull request #185 from meilisearch/serde-schema

Implement De/Serialize on schema

meilidb-data/src/database/index/main_index.rs

@@ -19,7 +19,7 @@ impl MainIndex {
     pub fn schema(&self) -> Result<Option<Schema>, Error> {
         match self.0.get(SCHEMA_KEY)? {
             Some(bytes) => {
-                let schema = Schema::read_from_bin(bytes.as_ref())?;
+                let schema = bincode::deserialize_from(bytes.as_ref())?;
                 Ok(Some(schema))
             },
             None => Ok(None),
@@ -27,8 +27,7 @@ impl MainIndex {
     }
 
     pub fn set_schema(&self, schema: &Schema) -> Result<(), Error> {
-        let mut bytes = Vec::new();
+        let bytes = bincode::serialize(schema)?;
-        schema.write_to_bin(&mut bytes)?;
         self.0.insert(SCHEMA_KEY, bytes)?;
         Ok(())
     }

meilidb-schema/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2018"
 
 [dependencies]
 bincode = "1.1.2"
-linked-hash-map = { version = "0.5.2", features = ["serde_impl"] }
+indexmap = { version = "1.1.0", features = ["serde-1"] }
 serde = { version = "1.0.91", features = ["derive"] }
 serde_json = { version = "1.0.39", features = ["preserve_order"] }
 toml = { version = "0.5.0", features = ["preserve_order"] }

meilidb-schema/src/lib.rs

@@ -6,7 +6,7 @@ use std::ops::BitOr;
 use std::sync::Arc;
 
 use serde::{Serialize, Deserialize};
-use linked_hash_map::LinkedHashMap;
+use indexmap::IndexMap;
 
 pub const DISPLAYED: SchemaProps = SchemaProps { displayed: true,  indexed: false, ranked: false };
 pub const INDEXED: SchemaProps   = SchemaProps { displayed: false, indexed: true,  ranked: false };
@@ -53,14 +53,14 @@ impl BitOr for SchemaProps {
 #[derive(Serialize, Deserialize)]
 pub struct SchemaBuilder {
     identifier: String,
-    attributes: LinkedHashMap<String, SchemaProps>,
+    attributes: IndexMap<String, SchemaProps>,
 }
 
 impl SchemaBuilder {
     pub fn with_identifier<S: Into<String>>(name: S) -> SchemaBuilder {
         SchemaBuilder {
             identifier: name.into(),
-            attributes: LinkedHashMap::new(),
+            attributes: IndexMap::new(),
         }
     }
 
@@ -99,62 +99,20 @@ struct InnerSchema {
 }
 
 impl Schema {
-    pub fn from_toml<R: Read>(mut reader: R) -> Result<Schema, Box<dyn Error>> {
+    fn to_builder(&self) -> SchemaBuilder {
-        let mut buffer = Vec::new();
-        reader.read_to_end(&mut buffer)?;
-        let builder: SchemaBuilder = toml::from_slice(&buffer)?;
-        Ok(builder.build())
-    }
-
-    pub fn to_toml<W: Write>(&self, mut writer: W) -> Result<(), Box<dyn Error>> {
         let identifier = self.inner.identifier.clone();
         let attributes = self.attributes_ordered();
-        let builder = SchemaBuilder { identifier, attributes };
+        SchemaBuilder { identifier, attributes }
-
-        let string = toml::to_string_pretty(&builder)?;
-        writer.write_all(string.as_bytes())?;
-
-        Ok(())
     }
 
-    pub fn from_json<R: Read>(mut reader: R) -> Result<Schema, Box<dyn Error>> {
+    fn attributes_ordered(&self) -> IndexMap<String, SchemaProps> {
-        let mut buffer = Vec::new();
-        reader.read_to_end(&mut buffer)?;
-        let builder: SchemaBuilder = serde_json::from_slice(&buffer)?;
-        Ok(builder.build())
-    }
-
-    pub fn to_json<W: Write>(&self, mut writer: W) -> Result<(), Box<dyn Error>> {
-        let identifier = self.inner.identifier.clone();
-        let attributes = self.attributes_ordered();
-        let builder = SchemaBuilder { identifier, attributes };
-        let string = serde_json::to_string_pretty(&builder)?;
-        writer.write_all(string.as_bytes())?;
-
-        Ok(())
-    }
-
-    pub fn read_from_bin<R: Read>(reader: R) -> bincode::Result<Schema> {
-        let builder: SchemaBuilder = bincode::deserialize_from(reader)?;
-        Ok(builder.build())
-    }
-
-    pub fn write_to_bin<W: Write>(&self, writer: W) -> bincode::Result<()> {
-        let identifier = self.inner.identifier.clone();
-        let attributes = self.attributes_ordered();
-        let builder = SchemaBuilder { identifier, attributes };
-
-        bincode::serialize_into(writer, &builder)
-    }
-
-    fn attributes_ordered(&self) -> LinkedHashMap<String, SchemaProps> {
         let mut ordered = BTreeMap::new();
         for (name, attr) in &self.inner.attrs {
             let (_, props) = self.inner.props[attr.0 as usize];
             ordered.insert(attr.0, (name, props));
         }
 
-        let mut attributes = LinkedHashMap::with_capacity(ordered.len());
+        let mut attributes = IndexMap::with_capacity(ordered.len());
         for (_, (name, props)) in ordered {
             attributes.insert(name.clone(), props);
         }
@@ -189,6 +147,23 @@ impl Schema {
     }
 }
 
+impl Serialize for Schema {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where S: serde::ser::Serializer,
+    {
+        self.to_builder().serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for Schema {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where D: serde::de::Deserializer<'de>,
+    {
+        let builder = SchemaBuilder::deserialize(deserializer)?;
+        Ok(builder.build())
+    }
+}
+
 #[derive(Serialize, Deserialize)]
 #[derive(Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)]
 pub struct SchemaAttr(pub u16);
@@ -235,9 +210,8 @@ mod tests {
         let schema = builder.build();
 
         let mut buffer = Vec::new();
-
+        bincode::serialize_into(&mut buffer, &schema)?;
-        schema.write_to_bin(&mut buffer)?;
+        let schema2 = bincode::deserialize_from(buffer.as_slice())?;
-        let schema2 = Schema::read_from_bin(buffer.as_slice())?;
 
         assert_eq!(schema, schema2);
 
@@ -252,10 +226,9 @@ mod tests {
         builder.new_attribute("gamma", INDEXED);
         let schema = builder.build();
 
-        let mut buffer = Vec::new();
+        let buffer = toml::to_vec(&schema)?;
-        schema.to_toml(&mut buffer)?;
+        let schema2 = toml::from_slice(buffer.as_slice())?;
 
-        let schema2 = Schema::from_toml(buffer.as_slice())?;
         assert_eq!(schema, schema2);
 
         let data = r#"
@@ -271,7 +244,7 @@ mod tests {
             [attributes."gamma"]
             indexed = true
         "#;
-        let schema2 = Schema::from_toml(data.as_bytes())?;
+        let schema2 = toml::from_str(data)?;
         assert_eq!(schema, schema2);
 
         Ok(())
@@ -285,10 +258,9 @@ mod tests {
         builder.new_attribute("gamma", INDEXED);
         let schema = builder.build();
 
-        let mut buffer = Vec::new();
+        let buffer = serde_json::to_vec(&schema)?;
-        schema.to_json(&mut buffer)?;
+        let schema2 = serde_json::from_slice(buffer.as_slice())?;
 
-        let schema2 = Schema::from_json(buffer.as_slice())?;
         assert_eq!(schema, schema2);
 
         let data = r#"
@@ -307,7 +279,7 @@ mod tests {
                     }
                 }
             }"#;
-        let schema2 = Schema::from_json(data.as_bytes())?;
+        let schema2 = serde_json::from_str(data)?;
         assert_eq!(schema, schema2);
 
         Ok(())

meilidb/Cargo.toml

@@ -13,8 +13,8 @@ meilidb-schema = { path = "../meilidb-schema", version = "0.1.0" }
 csv = "1.0.7"
 diskus = "0.5.0"
 env_logger = "0.6.1"
+indexmap = { version = "1.1.0", features = ["serde-1"] }
 jemallocator = "0.1.9"
-linked-hash-map = "0.5.2"
 meilidb-core = { path = "../meilidb-core", version = "0.1.0" }
 quickcheck = "0.8.2"
 rand = "0.6.5"
@@ -26,3 +26,4 @@ structopt = "0.2.15"
 sysinfo = "0.8.4"
 tempfile = "3.0.7"
 termcolor = "1.0.4"
+toml = "0.5.3"

meilidb/examples/create-database.rs

@@ -6,7 +6,7 @@ use std::io::{self, BufRead, BufReader};
 use std::path::{Path, PathBuf};
 use std::time::Instant;
 use std::error::Error;
-use std::fs::File;
+use std::fs::{self, File};
 
 use diskus::Walk;
 use sysinfo::{SystemExt, ProcessExt};
@@ -181,8 +181,8 @@ fn main() -> Result<(), Box<dyn Error>> {
     let opt = Opt::from_args();
 
     let schema = {
-        let file = File::open(&opt.schema_path)?;
+        let string = fs::read_to_string(&opt.schema_path)?;
-        Schema::from_toml(file)?
+        toml::from_str(&string)?
     };
 
     let stop_words = match opt.stop_words {

meilidb/examples/query-database.rs

@@ -9,7 +9,7 @@ use std::iter::FromIterator;
 use std::path::PathBuf;
 use std::time::{Instant, Duration};
 
-use linked_hash_map::LinkedHashMap;
+use indexmap::IndexMap;
 use rustyline::{Editor, Config};
 use structopt::StructOpt;
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
@@ -39,7 +39,7 @@ pub struct Opt {
     pub char_context: usize,
 }
 
-type Document = LinkedHashMap<String, String>;
+type Document = IndexMap<String, String>;
 
 fn display_highlights(text: &str, ranges: &[usize]) -> io::Result<()> {
     let mut stdout = StandardStream::stdout(ColorChoice::Always);