From 994a0e78f1f87f18c3c1a0a4d8e34ef270132e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 29 May 2019 15:26:18 +0200 Subject: [PATCH] feat: Move the Schema to its own workspace crate --- Cargo.toml | 1 + meilidb-data/Cargo.toml | 5 ++--- meilidb-data/src/database/documents_index.rs | 4 ++-- meilidb-data/src/database/index.rs | 2 +- meilidb-data/src/database/main_index.rs | 3 ++- meilidb-data/src/database/mod.rs | 3 +-- meilidb-data/src/document_attr_key.rs | 2 +- meilidb-data/src/indexer.rs | 3 +-- meilidb-data/src/lib.rs | 2 -- meilidb-data/src/ranked_map.rs | 3 ++- meilidb-data/src/serde/deserializer.rs | 2 +- meilidb-data/src/serde/indexer.rs | 2 +- meilidb-data/src/serde/mod.rs | 2 +- meilidb-data/src/serde/serializer.rs | 2 +- meilidb-data/tests/updates.rs | 4 ++-- meilidb-schema/Cargo.toml | 12 ++++++++++++ .../src/schema.rs => meilidb-schema/src/lib.rs | 0 meilidb/Cargo.toml | 1 + meilidb/examples/create-database.rs | 3 ++- meilidb/examples/query-database.rs | 2 +- meilidb/src/main.rs | 9 +++++---- meilidb/src/sort_by_attr.rs | 6 +++--- 22 files changed, 43 insertions(+), 30 deletions(-) create mode 100644 meilidb-schema/Cargo.toml rename meilidb-data/src/schema.rs => meilidb-schema/src/lib.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 69297052b..84a45aa9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "meilidb", "meilidb-core", "meilidb-data", + "meilidb-schema", "meilidb-tokenizer", ] diff --git a/meilidb-data/Cargo.toml b/meilidb-data/Cargo.toml index 88c917017..414cec7c7 100644 --- a/meilidb-data/Cargo.toml +++ b/meilidb-data/Cargo.toml @@ -9,15 +9,13 @@ arc-swap = "0.3.11" bincode = "1.1.2" deunicode = "1.0.0" hashbrown = { version = "0.2.2", features = ["serde"] } -linked-hash-map = { version = "0.5.2", features = ["serde_impl"] } meilidb-core = { path = "../meilidb-core", version = "0.1.0" } meilidb-tokenizer = { path = "../meilidb-tokenizer", version = "0.1.0" } +meilidb-schema = { path = "../meilidb-schema", version = "0.1.0" } ordered-float = { version = "1.0.2", features = ["serde"] } sdset = "0.3.2" serde = { version = "1.0.91", features = ["derive"] } -serde_json = { version = "1.0.39", features = ["preserve_order"] } rocksdb = { version = "0.12.2", default-features = false } -toml = { version = "0.5.0", features = ["preserve_order"] } zerocopy = "0.2.2" [dependencies.rmp-serde] @@ -30,3 +28,4 @@ branch = "arc-byte-slice" [dev-dependencies] tempfile = "3.0.7" +serde_json = "1.0.39" diff --git a/meilidb-data/src/database/documents_index.rs b/meilidb-data/src/database/documents_index.rs index 60420a8a9..361f9facb 100644 --- a/meilidb-data/src/database/documents_index.rs +++ b/meilidb-data/src/database/documents_index.rs @@ -1,11 +1,11 @@ use std::convert::TryInto; use meilidb_core::DocumentId; +use meilidb_schema::SchemaAttr; use rocksdb::DBVector; use crate::database::raw_index::InnerRawIndex; use crate::document_attr_key::DocumentAttrKey; -use crate::schema::SchemaAttr; #[derive(Clone)] pub struct DocumentsIndex(pub(crate) InnerRawIndex); @@ -52,7 +52,7 @@ impl DocumentsIndex { let from = rocksdb::IteratorMode::Start; let iterator = self.0.iterator(from)?; - for (key, value) in iterator { + for (key, _) in iterator { let slice = key.as_ref().try_into().unwrap(); let document_id = DocumentAttrKey::from_be_bytes(slice).document_id; diff --git a/meilidb-data/src/database/index.rs b/meilidb-data/src/database/index.rs index fce37ace6..4cc6d7acb 100644 --- a/meilidb-data/src/database/index.rs +++ b/meilidb-data/src/database/index.rs @@ -5,11 +5,11 @@ use std::sync::Arc; use arc_swap::{ArcSwap, Lease}; use meilidb_core::criterion::Criteria; use meilidb_core::{DocIndex, Store, DocumentId, QueryBuilder}; +use meilidb_schema::Schema; use rmp_serde::decode::Error as RmpError; use serde::de; use crate::ranked_map::RankedMap; -use crate::schema::Schema; use crate::serde::Deserializer; use super::{Error, CustomSettings}; diff --git a/meilidb-data/src/database/main_index.rs b/meilidb-data/src/database/main_index.rs index 4625450c5..7b3b98479 100644 --- a/meilidb-data/src/database/main_index.rs +++ b/meilidb-data/src/database/main_index.rs @@ -1,8 +1,9 @@ use std::sync::Arc; +use meilidb_schema::Schema; + use crate::database::raw_index::InnerRawIndex; use crate::ranked_map::RankedMap; -use crate::schema::Schema; use super::Error; diff --git a/meilidb-data/src/database/mod.rs b/meilidb-data/src/database/mod.rs index 3c66db026..b9df6fc0b 100644 --- a/meilidb-data/src/database/mod.rs +++ b/meilidb-data/src/database/mod.rs @@ -2,8 +2,7 @@ use std::collections::hash_map::Entry; use std::collections::{HashSet, HashMap}; use std::path::Path; use std::sync::{Arc, RwLock}; - -use crate::Schema; +use meilidb_schema::Schema; mod custom_settings; mod docs_words_index; diff --git a/meilidb-data/src/document_attr_key.rs b/meilidb-data/src/document_attr_key.rs index d8cb0bc8d..2664e6e9a 100644 --- a/meilidb-data/src/document_attr_key.rs +++ b/meilidb-data/src/document_attr_key.rs @@ -1,5 +1,5 @@ use meilidb_core::DocumentId; -use crate::schema::SchemaAttr; +use meilidb_schema::SchemaAttr; #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct DocumentAttrKey { diff --git a/meilidb-data/src/indexer.rs b/meilidb-data/src/indexer.rs index 25c71d2de..a4dc52f8a 100644 --- a/meilidb-data/src/indexer.rs +++ b/meilidb-data/src/indexer.rs @@ -3,11 +3,10 @@ use std::convert::TryFrom; use deunicode::deunicode_with_tofu; use meilidb_core::{DocumentId, DocIndex}; +use meilidb_schema::SchemaAttr; use meilidb_tokenizer::{is_cjk, Tokenizer, SeqTokenizer, Token}; use sdset::SetBuf; -use crate::SchemaAttr; - type Word = Vec; // TODO make it be a SmallVec pub struct Indexer { diff --git a/meilidb-data/src/lib.rs b/meilidb-data/src/lib.rs index 520a3e6e2..cb8a7dc9f 100644 --- a/meilidb-data/src/lib.rs +++ b/meilidb-data/src/lib.rs @@ -4,10 +4,8 @@ mod indexer; mod number; mod ranked_map; mod serde; -pub mod schema; pub use rocksdb; pub use self::database::{Database, Index, CustomSettings}; pub use self::number::Number; pub use self::ranked_map::RankedMap; -pub use self::schema::{Schema, SchemaAttr}; diff --git a/meilidb-data/src/ranked_map.rs b/meilidb-data/src/ranked_map.rs index 609cee954..96816613a 100644 --- a/meilidb-data/src/ranked_map.rs +++ b/meilidb-data/src/ranked_map.rs @@ -2,8 +2,9 @@ use std::io::{Read, Write}; use hashbrown::HashMap; use meilidb_core::DocumentId; +use meilidb_schema::SchemaAttr; -use crate::{SchemaAttr, Number}; +use crate::Number; #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct RankedMap(HashMap<(DocumentId, SchemaAttr), Number>); diff --git a/meilidb-data/src/serde/deserializer.rs b/meilidb-data/src/serde/deserializer.rs index 5c7baec38..549fafa90 100644 --- a/meilidb-data/src/serde/deserializer.rs +++ b/meilidb-data/src/serde/deserializer.rs @@ -2,12 +2,12 @@ use std::collections::HashSet; use std::io::Cursor; use meilidb_core::DocumentId; +use meilidb_schema::SchemaAttr; use rmp_serde::decode::{Deserializer as RmpDeserializer, ReadReader}; use rmp_serde::decode::{Error as RmpError}; use serde::{de, forward_to_deserialize_any}; use crate::database::Index; -use crate::SchemaAttr; pub struct Deserializer<'a> { pub document_id: DocumentId, diff --git a/meilidb-data/src/serde/indexer.rs b/meilidb-data/src/serde/indexer.rs index f9445dca4..b06d48322 100644 --- a/meilidb-data/src/serde/indexer.rs +++ b/meilidb-data/src/serde/indexer.rs @@ -1,9 +1,9 @@ use meilidb_core::DocumentId; +use meilidb_schema::SchemaAttr; use serde::ser; use serde::Serialize; use crate::indexer::Indexer as RawIndexer; -use crate::schema::SchemaAttr; use super::{SerializerError, ConvertToString}; pub struct Indexer<'a> { diff --git a/meilidb-data/src/serde/mod.rs b/meilidb-data/src/serde/mod.rs index 0fe5918cc..b536c63cc 100644 --- a/meilidb-data/src/serde/mod.rs +++ b/meilidb-data/src/serde/mod.rs @@ -26,11 +26,11 @@ use std::collections::BTreeMap; use std::{fmt, error::Error}; use meilidb_core::DocumentId; +use meilidb_schema::SchemaAttr; use rmp_serde::encode::Error as RmpError; use serde::ser; use crate::number::ParseNumberError; -use crate::schema::SchemaAttr; #[derive(Debug)] pub enum SerializerError { diff --git a/meilidb-data/src/serde/serializer.rs b/meilidb-data/src/serde/serializer.rs index d201b3b4d..96b9b3df9 100644 --- a/meilidb-data/src/serde/serializer.rs +++ b/meilidb-data/src/serde/serializer.rs @@ -1,9 +1,9 @@ use meilidb_core::DocumentId; +use meilidb_schema::Schema; use serde::ser; use crate::indexer::Indexer as RawIndexer; use crate::ranked_map::RankedMap; -use crate::schema::Schema; use super::{RamDocumentStore, SerializerError, ConvertToString, ConvertToNumber, Indexer}; pub struct Serializer<'a> { diff --git a/meilidb-data/tests/updates.rs b/meilidb-data/tests/updates.rs index 393b2ecba..e5534f23e 100644 --- a/meilidb-data/tests/updates.rs +++ b/meilidb-data/tests/updates.rs @@ -1,6 +1,6 @@ use serde_json::json; -use meilidb_data::{Database, Schema}; -use meilidb_data::schema::{SchemaBuilder, STORED, INDEXED}; +use meilidb_data::Database; +use meilidb_schema::{Schema, SchemaBuilder, STORED, INDEXED}; fn simple_schema() -> Schema { let mut builder = SchemaBuilder::with_identifier("objectId"); diff --git a/meilidb-schema/Cargo.toml b/meilidb-schema/Cargo.toml new file mode 100644 index 000000000..86f772452 --- /dev/null +++ b/meilidb-schema/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "meilidb-schema" +version = "0.1.0" +authors = ["Kerollmops "] +edition = "2018" + +[dependencies] +bincode = "1.1.2" +linked-hash-map = { version = "0.5.2", features = ["serde_impl"] } +serde = { version = "1.0.91", features = ["derive"] } +serde_json = { version = "1.0.39", features = ["preserve_order"] } +toml = { version = "0.5.0", features = ["preserve_order"] } diff --git a/meilidb-data/src/schema.rs b/meilidb-schema/src/lib.rs similarity index 100% rename from meilidb-data/src/schema.rs rename to meilidb-schema/src/lib.rs diff --git a/meilidb/Cargo.toml b/meilidb/Cargo.toml index 1b568cf1c..5c80a6d26 100644 --- a/meilidb/Cargo.toml +++ b/meilidb/Cargo.toml @@ -7,6 +7,7 @@ authors = ["Kerollmops "] [dependencies] meilidb-core = { path = "../meilidb-core", version = "0.1.0" } meilidb-data = { path = "../meilidb-data", version = "0.1.0" } +meilidb-schema = { path = "../meilidb-schema", version = "0.1.0" } serde = { version = "1.0.91" , features = ["derive"] } serde_json = "1.0.39" tempfile = "3.0.7" diff --git a/meilidb/examples/create-database.rs b/meilidb/examples/create-database.rs index 1518b4298..5623fe4bb 100644 --- a/meilidb/examples/create-database.rs +++ b/meilidb/examples/create-database.rs @@ -14,7 +14,8 @@ use sysinfo::{SystemExt, ProcessExt}; use serde::{Serialize, Deserialize}; use structopt::StructOpt; -use meilidb_data::{Database, Schema}; +use meilidb_data::Database; +use meilidb_schema::Schema; #[derive(Debug, StructOpt)] pub struct Opt { diff --git a/meilidb/examples/query-database.rs b/meilidb/examples/query-database.rs index 4fd529c27..599dc8fa6 100644 --- a/meilidb/examples/query-database.rs +++ b/meilidb/examples/query-database.rs @@ -13,8 +13,8 @@ use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; use structopt::StructOpt; use meilidb_core::Match; -use meilidb_data::schema::SchemaAttr; use meilidb_data::Database; +use meilidb_schema::SchemaAttr; #[derive(Debug, StructOpt)] pub struct Opt { diff --git a/meilidb/src/main.rs b/meilidb/src/main.rs index 822d941b5..dd074b956 100644 --- a/meilidb/src/main.rs +++ b/meilidb/src/main.rs @@ -2,12 +2,13 @@ use std::collections::HashMap; +use meilidb_data::Database; +use meilidb_schema::Schema; use serde::{Deserialize, Serialize}; -use tide::querystring::ExtractQuery; -use tide::http::status::StatusCode; -use tide::{error::ResultExt, response, App, Context, EndpointResult}; use serde_json::Value; -use meilidb_data::{Database, Schema}; +use tide::http::status::StatusCode; +use tide::querystring::ExtractQuery; +use tide::{error::ResultExt, response, App, Context, EndpointResult}; #[derive(Debug, Serialize, Deserialize, Clone)] struct SearchQuery { diff --git a/meilidb/src/sort_by_attr.rs b/meilidb/src/sort_by_attr.rs index 6a630bedf..83577df13 100644 --- a/meilidb/src/sort_by_attr.rs +++ b/meilidb/src/sort_by_attr.rs @@ -2,9 +2,9 @@ use std::cmp::Ordering; use std::error::Error; use std::fmt; -use meilidb_core::criterion::Criterion; -use meilidb_core::RawDocument; -use meilidb_data::{Schema, SchemaAttr, RankedMap}; +use meilidb_core::{criterion::Criterion, RawDocument}; +use meilidb_data::RankedMap; +use meilidb_schema::{Schema, SchemaAttr}; /// An helper struct that permit to sort documents by /// some of their stored attributes.