From 37578ed74f526014a04d57fef878921310553de7 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 12 Feb 2019 17:20:47 +0100 Subject: [PATCH 1/8] feat: store config into database --- src/database/config.rs | 29 +++++++++++++++++++++++++++++ src/database/mod.rs | 30 ++++++++++++++++++++++++++++++ src/database/view.rs | 7 +++++-- 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 src/database/config.rs diff --git a/src/database/config.rs b/src/database/config.rs new file mode 100644 index 000000000..2150566a8 --- /dev/null +++ b/src/database/config.rs @@ -0,0 +1,29 @@ +use std::collections::{HashSet, HashMap}; + +use serde_derive::{Serialize, Deserialize}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum RankingOrdering { + Asc, + Dsc +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Config { + stop_words: Option>, + ranking_order: Option>, + distinct_field: Option, + ranking_rules: Option>, +} + + +impl Config { + pub(crate) fn default() -> Config { + Config { + stop_words: None, + ranking_order: None, + distinct_field: None, + ranking_rules: None, + } + } +} diff --git a/src/database/mod.rs b/src/database/mod.rs index 5d526dcd7..8a032080e 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -23,6 +23,7 @@ use crate::DocumentId; use self::update::{ReadIndexEvent, ReadRankedMapEvent}; +pub use self::config::Config; pub use self::document_key::{DocumentKey, DocumentKeyAttr}; pub use self::view::{DatabaseView, DocumentIter}; pub use self::update::Update; @@ -31,12 +32,15 @@ pub use self::schema::Schema; pub use self::index::Index; pub use self::number::{Number, ParseNumberError}; + pub type RankedMap = HashMap<(DocumentId, SchemaAttr), Number>; const DATA_INDEX: &[u8] = b"data-index"; const DATA_RANKED_MAP: &[u8] = b"data-ranked-map"; const DATA_SCHEMA: &[u8] = b"data-schema"; +const CONFIG: &[u8] = b"config"; +pub mod config; pub mod schema; pub(crate) mod index; mod number; @@ -104,6 +108,15 @@ where D: Deref, } } +fn retrieve_config(snapshot: &Snapshot) -> Result> +where D: Deref, +{ + match snapshot.get(CONFIG)? { + Some(vector) => Ok(bincode::deserialize(&*vector)?), + None => Ok(Config::default()), + } +} + fn merge_indexes(existing: Option<&[u8]>, operands: &mut MergeOperands) -> Vec { use self::update::ReadIndexEvent::{self, *}; use self::update::WriteIndexEvent; @@ -265,6 +278,17 @@ impl DatabaseIndex { fn view(&self) -> Arc>> { self.view.load() } + + fn update_config(&self, config: Config) -> Result>>, Box>{ + let data = bincode::serialize(&config)?; + self.db.put(CONFIG, &data)?; + + let snapshot = Snapshot::new(self.db.clone()); + let view = Arc::new(DatabaseView::new(snapshot)?); + self.view.set(view.clone()); + + Ok(view) + } } impl Drop for DatabaseIndex { @@ -369,6 +393,12 @@ impl Database { Ok(index_guard.val().view()) } + pub fn update_config(&self, index: &str, config: Config) -> Result>>, Box>{ + let index_guard = self.indexes.get(index).ok_or("Index not found")?; + + Ok(index_guard.val().update_config(config)?) + } + } #[cfg(test)] diff --git a/src/database/view.rs b/src/database/view.rs index 74e4ef002..3cee6db4e 100644 --- a/src/database/view.rs +++ b/src/database/view.rs @@ -7,13 +7,14 @@ use rocksdb::rocksdb_options::{ReadOptions, EnvOptions, ColumnFamilyOptions}; use rocksdb::rocksdb::{DB, DBVector, Snapshot, SeekKey, SstFileWriter}; use serde::de::DeserializeOwned; -use crate::database::{retrieve_data_schema, retrieve_data_index, retrieve_data_ranked_map}; +use crate::database::{retrieve_data_schema, retrieve_data_index, retrieve_data_ranked_map, retrieve_config}; use crate::database::serde::deserializer::Deserializer; use crate::database::{DocumentKey, DocumentKeyAttr}; use crate::rank::{QueryBuilder, FilterFunc}; use crate::database::schema::Schema; use crate::database::index::Index; use crate::database::RankedMap; +use crate::database::Config; use crate::DocumentId; pub struct DatabaseView @@ -23,6 +24,7 @@ where D: Deref index: Index, ranked_map: RankedMap, schema: Schema, + config: Config, } impl DatabaseView @@ -32,7 +34,8 @@ where D: Deref let schema = retrieve_data_schema(&snapshot)?; let index = retrieve_data_index(&snapshot)?; let ranked_map = retrieve_data_ranked_map(&snapshot)?; - Ok(DatabaseView { snapshot, index, ranked_map, schema }) + let config = retrieve_config(&snapshot)?; + Ok(DatabaseView { snapshot, index, ranked_map, schema, config }) } pub fn schema(&self) -> &Schema { From d5119db165e2c7e2d938da4eeee62f4ab7760d19 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 12 Feb 2019 18:41:51 +0100 Subject: [PATCH 2/8] feat: Allow to retrieve config from Database and DatabaseView --- src/database/mod.rs | 12 +++++++++++- src/database/view.rs | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/database/mod.rs b/src/database/mod.rs index 8a032080e..7a2982fb5 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -279,13 +279,17 @@ impl DatabaseIndex { self.view.load() } + fn get_config(&self) -> Config { + self.view().config().clone() + } + fn update_config(&self, config: Config) -> Result>>, Box>{ let data = bincode::serialize(&config)?; self.db.put(CONFIG, &data)?; let snapshot = Snapshot::new(self.db.clone()); let view = Arc::new(DatabaseView::new(snapshot)?); - self.view.set(view.clone()); + self.view.store(view.clone()); Ok(view) } @@ -393,6 +397,12 @@ impl Database { Ok(index_guard.val().view()) } + pub fn get_config(&self, index: &str) -> Result> { + let index_guard = self.indexes.get(index).ok_or("Index not found")?; + + Ok(index_guard.val().get_config()) + } + pub fn update_config(&self, index: &str, config: Config) -> Result>>, Box>{ let index_guard = self.indexes.get(index).ok_or("Index not found")?; diff --git a/src/database/view.rs b/src/database/view.rs index 3cee6db4e..81d14c3d5 100644 --- a/src/database/view.rs +++ b/src/database/view.rs @@ -58,6 +58,10 @@ where D: Deref &self.snapshot } + pub fn config(&self) -> &Config { + &self.config + } + pub fn get(&self, key: &[u8]) -> Result, Box> { Ok(self.snapshot.get(key)?) } From 482f750231945fd4ebe11055646bef0b62499b8d Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 12 Feb 2019 19:12:41 +0100 Subject: [PATCH 3/8] chore: Set config field pub --- src/database/config.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/database/config.rs b/src/database/config.rs index 2150566a8..ba6fd1d0a 100644 --- a/src/database/config.rs +++ b/src/database/config.rs @@ -10,10 +10,10 @@ pub enum RankingOrdering { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Config { - stop_words: Option>, - ranking_order: Option>, - distinct_field: Option, - ranking_rules: Option>, + pub stop_words: Option>, + pub ranking_order: Option>, + pub distinct_field: Option, + pub ranking_rules: Option>, } From 4721da1679a563dc686dbfe61f7fdf7ef537556d Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 14 Feb 2019 15:44:11 +0100 Subject: [PATCH 4/8] feat: Add access key on config --- src/database/config.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/database/config.rs b/src/database/config.rs index ba6fd1d0a..653ef1878 100644 --- a/src/database/config.rs +++ b/src/database/config.rs @@ -8,12 +8,20 @@ pub enum RankingOrdering { Dsc } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AccessToken { + pub token: String, + pub secret_key: String, +} + + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Config { pub stop_words: Option>, pub ranking_order: Option>, pub distinct_field: Option, pub ranking_rules: Option>, + pub access_token: Option, } @@ -24,6 +32,7 @@ impl Config { ranking_order: None, distinct_field: None, ranking_rules: None, + access_token: None, } } } From 55823c5d5d2e61ec6a233e16fd67f5c16aaebf5b Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 14 Feb 2019 16:36:03 +0100 Subject: [PATCH 5/8] feat: add admin key on config --- src/database/config.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/database/config.rs b/src/database/config.rs index 653ef1878..ef1b6ebb4 100644 --- a/src/database/config.rs +++ b/src/database/config.rs @@ -10,8 +10,9 @@ pub enum RankingOrdering { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AccessToken { - pub token: String, - pub secret_key: String, + pub read_key: String, + pub write_key: String, + pub admin_key: String, } From 1941cb16c09a1711ddd7487915cd27fe79da5bea Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Wed, 20 Feb 2019 11:04:22 +0100 Subject: [PATCH 6/8] feat: Add Config.update_with(_) method to merge 2 config --- src/database/config.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/database/config.rs b/src/database/config.rs index ef1b6ebb4..55bab6272 100644 --- a/src/database/config.rs +++ b/src/database/config.rs @@ -25,7 +25,6 @@ pub struct Config { pub access_token: Option, } - impl Config { pub(crate) fn default() -> Config { Config { @@ -36,4 +35,22 @@ impl Config { access_token: None, } } + + pub fn update_with(&mut self, new: Config) { + if let Some(stop_words) = new.stop_words { + self.stop_words = Some(stop_words); + }; + if let Some(ranking_order) = new.ranking_order { + self.ranking_order = Some(ranking_order); + }; + if let Some(distinct_field) = new.distinct_field { + self.distinct_field = Some(distinct_field); + }; + if let Some(ranking_rules) = new.ranking_rules { + self.ranking_rules = Some(ranking_rules); + }; + if let Some(access_token) = new.access_token { + self.access_token = Some(access_token); + }; + } } From 13309511b3fc99322df412651779acd650cf9641 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 21 Feb 2019 13:56:08 +0100 Subject: [PATCH 7/8] chore: Use serde derive lowercase on RankingOrdering --- src/database/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/config.rs b/src/database/config.rs index 55bab6272..251d5e93b 100644 --- a/src/database/config.rs +++ b/src/database/config.rs @@ -1,8 +1,8 @@ use std::collections::{HashSet, HashMap}; - use serde_derive::{Serialize, Deserialize}; #[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum RankingOrdering { Asc, Dsc From 9437cecf875ba46f69027680c114bdf6470dc783 Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Thu, 21 Feb 2019 13:58:41 +0100 Subject: [PATCH 8/8] chore: Use Default derive on Config struct --- src/database/config.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/database/config.rs b/src/database/config.rs index 251d5e93b..491cdba93 100644 --- a/src/database/config.rs +++ b/src/database/config.rs @@ -16,7 +16,7 @@ pub struct AccessToken { } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Config { pub stop_words: Option>, pub ranking_order: Option>, @@ -26,16 +26,6 @@ pub struct Config { } impl Config { - pub(crate) fn default() -> Config { - Config { - stop_words: None, - ranking_order: None, - distinct_field: None, - ranking_rules: None, - access_token: None, - } - } - pub fn update_with(&mut self, new: Config) { if let Some(stop_words) = new.stop_words { self.stop_words = Some(stop_words);