snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-27 04:25:06 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge pull request #185 from meilisearch/serde-schema

Browse Source 
Implement De/Serialize on schema
This commit is contained in:
Clément Renault 2019-09-16 15:18:02 +02:00 committed by GitHub
commit 8d24e54fa1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 42 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
meilidb-data/src/database/index/main_index.rs
View File

@ -19,7 +19,7 @@ impl MainIndex {
pub fn schema(&self) -> Result<Option<Schema>, Error> {
match self.0.get(SCHEMA_KEY)? {
Some(bytes) => {
let schema = Schema::read_from_bin(bytes.as_ref())?;
let schema = bincode::deserialize_from(bytes.as_ref())?;
Ok(Some(schema))
},
None => Ok(None),
@ -27,8 +27,7 @@ impl MainIndex {
}
pub fn set_schema(&self, schema: &Schema) -> Result<(), Error> {
let mut bytes = Vec::new();
schema.write_to_bin(&mut bytes)?;
let bytes = bincode::serialize(schema)?;
self.0.insert(SCHEMA_KEY, bytes)?;
Ok(())
}

2
meilidb-schema/Cargo.toml
View File

@ -6,7 +6,7 @@ edition = "2018"
[dependencies]
bincode = "1.1.2"
linked-hash-map = { version = "0.5.2", features = ["serde_impl"] }
indexmap = { version = "1.1.0", features = ["serde-1"] }
serde = { version = "1.0.91", features = ["derive"] }
serde_json = { version = "1.0.39", features = ["preserve_order"] }
toml = { version = "0.5.0", features = ["preserve_order"] }

92
meilidb-schema/src/lib.rs
View File

@ -6,7 +6,7 @@ use std::ops::BitOr;
use std::sync::Arc;
use serde::{Serialize, Deserialize};
use linked_hash_map::LinkedHashMap;
use indexmap::IndexMap;
pub const DISPLAYED: SchemaProps = SchemaProps { displayed: true, indexed: false, ranked: false };
pub const INDEXED: SchemaProps = SchemaProps { displayed: false, indexed: true, ranked: false };
@ -53,14 +53,14 @@ impl BitOr for SchemaProps {
#[derive(Serialize, Deserialize)]
pub struct SchemaBuilder {
identifier: String,
attributes: LinkedHashMap<String, SchemaProps>,
attributes: IndexMap<String, SchemaProps>,
}
impl SchemaBuilder {
pub fn with_identifier<S: Into<String>>(name: S) -> SchemaBuilder {
SchemaBuilder {
identifier: name.into(),
attributes: LinkedHashMap::new(),
attributes: IndexMap::new(),
}
}
@ -99,62 +99,20 @@ struct InnerSchema {
}
impl Schema {
pub fn from_toml<R: Read>(mut reader: R) -> Result<Schema, Box<dyn Error>> {
let mut buffer = Vec::new();
reader.read_to_end(&mut buffer)?;
let builder: SchemaBuilder = toml::from_slice(&buffer)?;
Ok(builder.build())
}
pub fn to_toml<W: Write>(&self, mut writer: W) -> Result<(), Box<dyn Error>> {
fn to_builder(&self) -> SchemaBuilder {
let identifier = self.inner.identifier.clone();
let attributes = self.attributes_ordered();
let builder = SchemaBuilder { identifier, attributes };
let string = toml::to_string_pretty(&builder)?;
writer.write_all(string.as_bytes())?;
Ok(())
SchemaBuilder { identifier, attributes }
}
pub fn from_json<R: Read>(mut reader: R) -> Result<Schema, Box<dyn Error>> {
let mut buffer = Vec::new();
reader.read_to_end(&mut buffer)?;
let builder: SchemaBuilder = serde_json::from_slice(&buffer)?;
Ok(builder.build())
}
pub fn to_json<W: Write>(&self, mut writer: W) -> Result<(), Box<dyn Error>> {
let identifier = self.inner.identifier.clone();
let attributes = self.attributes_ordered();
let builder = SchemaBuilder { identifier, attributes };
let string = serde_json::to_string_pretty(&builder)?;
writer.write_all(string.as_bytes())?;
Ok(())
}
pub fn read_from_bin<R: Read>(reader: R) -> bincode::Result<Schema> {
let builder: SchemaBuilder = bincode::deserialize_from(reader)?;
Ok(builder.build())
}
pub fn write_to_bin<W: Write>(&self, writer: W) -> bincode::Result<()> {
let identifier = self.inner.identifier.clone();
let attributes = self.attributes_ordered();
let builder = SchemaBuilder { identifier, attributes };
bincode::serialize_into(writer, &builder)
}
fn attributes_ordered(&self) -> LinkedHashMap<String, SchemaProps> {
fn attributes_ordered(&self) -> IndexMap<String, SchemaProps> {
let mut ordered = BTreeMap::new();
for (name, attr) in &self.inner.attrs {
let (_, props) = self.inner.props[attr.0 as usize];
ordered.insert(attr.0, (name, props));
}
let mut attributes = LinkedHashMap::with_capacity(ordered.len());
let mut attributes = IndexMap::with_capacity(ordered.len());
for (_, (name, props)) in ordered {
attributes.insert(name.clone(), props);
}
@ -189,6 +147,23 @@ impl Schema {
}
}
impl Serialize for Schema {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: serde::ser::Serializer,
{
self.to_builder().serialize(serializer)
}
}
impl<'de> Deserialize<'de> for Schema {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where D: serde::de::Deserializer<'de>,
{
let builder = SchemaBuilder::deserialize(deserializer)?;
Ok(builder.build())
}
}
#[derive(Serialize, Deserialize)]
#[derive(Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)]
pub struct SchemaAttr(pub u16);
@ -235,9 +210,8 @@ mod tests {
let schema = builder.build();
let mut buffer = Vec::new();
schema.write_to_bin(&mut buffer)?;
let schema2 = Schema::read_from_bin(buffer.as_slice())?;
bincode::serialize_into(&mut buffer, &schema)?;
let schema2 = bincode::deserialize_from(buffer.as_slice())?;
assert_eq!(schema, schema2);
@ -252,10 +226,9 @@ mod tests {
builder.new_attribute("gamma", INDEXED);
let schema = builder.build();
let mut buffer = Vec::new();
schema.to_toml(&mut buffer)?;
let buffer = toml::to_vec(&schema)?;
let schema2 = toml::from_slice(buffer.as_slice())?;
let schema2 = Schema::from_toml(buffer.as_slice())?;
assert_eq!(schema, schema2);
let data = r#"
@ -271,7 +244,7 @@ mod tests {
[attributes."gamma"]
indexed = true
"#;
let schema2 = Schema::from_toml(data.as_bytes())?;
let schema2 = toml::from_str(data)?;
assert_eq!(schema, schema2);
Ok(())
@ -285,10 +258,9 @@ mod tests {
builder.new_attribute("gamma", INDEXED);
let schema = builder.build();
let mut buffer = Vec::new();
schema.to_json(&mut buffer)?;
let buffer = serde_json::to_vec(&schema)?;
let schema2 = serde_json::from_slice(buffer.as_slice())?;
let schema2 = Schema::from_json(buffer.as_slice())?;
assert_eq!(schema, schema2);
let data = r#"
@ -307,7 +279,7 @@ mod tests {
}
}
}"#;
let schema2 = Schema::from_json(data.as_bytes())?;
let schema2 = serde_json::from_str(data)?;
assert_eq!(schema, schema2);
Ok(())

3
meilidb/Cargo.toml
View File

@ -13,8 +13,8 @@ meilidb-schema = { path = "../meilidb-schema", version = "0.1.0" }
csv = "1.0.7"
diskus = "0.5.0"
env_logger = "0.6.1"
indexmap = { version = "1.1.0", features = ["serde-1"] }
jemallocator = "0.1.9"
linked-hash-map = "0.5.2"
meilidb-core = { path = "../meilidb-core", version = "0.1.0" }
quickcheck = "0.8.2"
rand = "0.6.5"
@ -26,3 +26,4 @@ structopt = "0.2.15"
sysinfo = "0.8.4"
tempfile = "3.0.7"
termcolor = "1.0.4"
toml = "0.5.3"

6
meilidb/examples/create-database.rs
View File

@ -6,7 +6,7 @@ use std::io::{self, BufRead, BufReader};
use std::path::{Path, PathBuf};
use std::time::Instant;
use std::error::Error;
use std::fs::File;
use std::fs::{self, File};
use diskus::Walk;
use sysinfo::{SystemExt, ProcessExt};
@ -181,8 +181,8 @@ fn main() -> Result<(), Box<dyn Error>> {
let opt = Opt::from_args();
let schema = {
let file = File::open(&opt.schema_path)?;
Schema::from_toml(file)?
let string = fs::read_to_string(&opt.schema_path)?;
toml::from_str(&string)?
};
let stop_words = match opt.stop_words {

4
meilidb/examples/query-database.rs
View File

@ -9,7 +9,7 @@ use std::iter::FromIterator;
use std::path::PathBuf;
use std::time::{Instant, Duration};
use linked_hash_map::LinkedHashMap;
use indexmap::IndexMap;
use rustyline::{Editor, Config};
use structopt::StructOpt;
use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
@ -39,7 +39,7 @@ pub struct Opt {
pub char_context: usize,
}
type Document = LinkedHashMap<String, String>;
type Document = IndexMap<String, String>;
fn display_highlights(text: &str, ranges: &[usize]) -> io::Result<()> {
let mut stdout = StandardStream::stdout(ColorChoice::Always);