feat: Introduce the QueryBuilder struct

src/blob/mod.rs

@@ -1,5 +1,5 @@
 mod merge;
-mod ops;
+pub mod ops;
 mod ops_indexed_value;
 mod positive_blob;
 mod negative_blob;

src/database.rs

@@ -0,0 +1,42 @@
+use std::error::Error;
+use std::ops::Deref;
+
+use ::rocksdb::rocksdb::{DB, Snapshot};
+
+use crate::index::schema::Schema;
+use crate::blob::PositiveBlob;
+use crate::DocumentId;
+
+pub trait Retrieve {
+    fn schema(&self) -> Result<Option<Schema>, Box<Error>>;
+    fn data_index(&self) -> Result<PositiveBlob, Box<Error>>;
+    fn get_documents<D>(&self, ids: &[DocumentId]) -> Result<Vec<D>, Box<Error>>;
+}
+
+impl<T> Retrieve for Snapshot<T>
+where T: Deref<Target=DB>,
+{
+    fn schema(&self) -> Result<Option<Schema>, Box<Error>> {
+        match self.deref().get(b"data-schema")? {
+            Some(value) => Ok(Some(Schema::read_from(&*value)?)),
+            None => Ok(None),
+        }
+    }
+
+    fn data_index(&self) -> Result<PositiveBlob, Box<Error>> {
+        match self.deref().get(b"data-index")? {
+            Some(value) => Ok(bincode::deserialize(&value)?),
+            None => Ok(PositiveBlob::default()),
+        }
+    }
+
+    fn get_documents<D>(&self, ids: &[DocumentId]) -> Result<Vec<D>, Box<Error>> {
+        if ids.is_empty() { return Ok(Vec::new()) }
+        let schema = match self.schema()? {
+            Some(schema) => schema,
+            None => return Err(String::from("BUG: could not find schema").into()),
+        };
+
+        unimplemented!()
+    }
+}

src/index/mod.rs

@@ -238,9 +238,9 @@ impl Index {
         let snapshot = self.database.snapshot();
 
         let index_key = Identifier::data().index().build();
-        let map = match snapshot.get(&index_key)? {
+        let blob = match snapshot.get(&index_key)? {
             Some(value) => bincode::deserialize(&value)?,
-            None => Vec::new(),
+            None => PositiveBlob::default(),
         };
 
         let mut automatons = Vec::new();
@@ -250,7 +250,7 @@ impl Index {
         }
 
         let config = Config {
-            map: map,
+            blob: blob,
             automatons: automatons,
             criteria: criterion::default(),
             distinct: ((), 1),

src/index/update/positive_update.rs

@@ -8,7 +8,7 @@ use crate::index::update::Update;
 use crate::index::identifier::Identifier;
 use crate::index::schema::{SchemaProps, Schema, SchemaAttr};
 use crate::tokenizer::TokenizerBuilder;
-use crate::blob::{BlobInfo, PositiveBlobBuilder};
+use crate::blob::PositiveBlobBuilder;
 use crate::{DocIndex, DocumentId};
 
 pub enum NewState {

src/lib.rs

@@ -3,13 +3,14 @@
 #[macro_use] extern crate lazy_static;
 #[macro_use] extern crate serde_derive;
 
-pub mod index;
+pub mod automaton;
 pub mod blob;
 pub mod data;
+pub mod database;
+pub mod index;
 pub mod rank;
-pub mod vec_read_only;
-pub mod automaton;
 pub mod tokenizer;
+pub mod vec_read_only;
 mod common_words;
 
 pub use self::tokenizer::Tokenizer;

src/rank/ranked_stream.rs

@@ -1,19 +1,24 @@
+use std::ops::{Deref, Range, RangeBounds};
 use std::collections::HashMap;
+use std::{mem, vec, str};
+use std::ops::Bound::*;
+use std::error::Error;
 use std::hash::Hash;
-use std::ops::Range;
 use std::rc::Rc;
-use std::{mem, vec};
 
 use fnv::FnvHashMap;
 use fst::Streamer;
 use group_by::GroupByMut;
+use ::rocksdb::rocksdb::{DB, Snapshot};
 
-use crate::automaton::{DfaExt, AutomatonExt};
+use crate::automaton::{self, DfaExt, AutomatonExt};
-use crate::index::Index;
+use crate::rank::criterion::{self, Criterion};
-use crate::blob::{Blob, Merge};
+use crate::blob::{PositiveBlob, Merge};
-use crate::rank::criterion::Criterion;
+use crate::blob::ops::Union;
-use crate::rank::Document;
 use crate::{Match, DocumentId};
+use crate::database::Retrieve;
+use crate::rank::Document;
+use crate::index::Index;
 
 fn clamp_range<T: Copy + Ord>(range: Range<T>, big: Range<T>) -> Range<T> {
     Range {
@@ -22,40 +27,58 @@ fn clamp_range<T: Copy + Ord>(range: Range<T>, big: Range<T>) -> Range<T> {
     }
 }
 
-pub struct Config<'a, C, F> {
+fn split_whitespace_automatons(query: &str) -> Vec<DfaExt> {
-    pub blobs: &'a [Blob],
+    let mut automatons = Vec::new();
-    pub automatons: Vec<DfaExt>,
+    for query in query.split_whitespace().map(str::to_lowercase) {
-    pub criteria: Vec<C>,
+        let lev = automaton::build_prefix_dfa(&query);
-    pub distinct: (F, usize),
+        automatons.push(lev);
+    }
+    automatons
 }
 
-pub struct RankedStream<'m, C, F> {
+pub struct QueryBuilder<T: Deref<Target=DB>, C> {
-    stream: crate::blob::Merge<'m>,
+    snapshot: Snapshot<T>,
-    automatons: Vec<Rc<DfaExt>>,
+    blob: PositiveBlob,
     criteria: Vec<C>,
-    distinct: (F, usize),
 }
 
-impl<'m, C, F> RankedStream<'m, C, F> {
+impl<T: Deref<Target=DB>> QueryBuilder<T, Box<dyn Criterion>> {
-    pub fn new(config: Config<'m, C, F>) -> Self {
+    pub fn new(snapshot: Snapshot<T>) -> Result<Self, Box<Error>> {
-        let automatons: Vec<_> = config.automatons.into_iter().map(Rc::new).collect();
+        QueryBuilder::with_criteria(snapshot, criterion::default())
-
-        RankedStream {
-            stream: Merge::with_automatons(automatons.clone(), config.blobs),
-            automatons: automatons,
-            criteria: config.criteria,
-            distinct: config.distinct,
-        }
     }
 }
 
-impl<'m, C, F> RankedStream<'m, C, F> {
+impl<T, C> QueryBuilder<T, C>
-    fn retrieve_all_documents(&mut self) -> Vec<Document> {
+where T: Deref<Target=DB>,
+{
+    pub fn with_criteria(snapshot: Snapshot<T>, criteria: Vec<C>) -> Result<Self, Box<Error>> {
+        let blob = snapshot.data_index()?;
+        Ok(QueryBuilder { snapshot, blob, criteria })
+    }
+
+    pub fn criteria(&mut self, criteria: Vec<C>) -> &mut Self {
+        self.criteria = criteria;
+        self
+    }
+
+    pub fn with_distinct<F>(self, function: F, size: usize) -> DistinctQueryBuilder<T, F, C> {
+        DistinctQueryBuilder {
+            snapshot: self.snapshot,
+            blob: self.blob,
+            criteria: self.criteria,
+            function: function,
+            size: size
+        }
+    }
+
+    fn query_all(&self, query: &str) -> Vec<Document> {
+        let automatons = split_whitespace_automatons(query);
+        let mut stream: Union = unimplemented!();
         let mut matches = FnvHashMap::default();
 
-        while let Some((string, indexed_values)) = self.stream.next() {
+        while let Some((string, indexed_values)) = stream.next() {
             for iv in indexed_values {
-                let automaton = &self.automatons[iv.index];
+                let automaton = &automatons[iv.index];
                 let distance = automaton.eval(string).to_u8();
                 let is_exact = distance == 0 && string.len() == automaton.query_len();
 
@@ -76,12 +99,12 @@ impl<'m, C, F> RankedStream<'m, C, F> {
     }
 }
 
-impl<'a, C, F> RankedStream<'a, C, F>
+impl<T, C> QueryBuilder<T, C>
-where C: Criterion
+where T: Deref<Target=DB>,
+      C: Criterion,
 {
-    // TODO don't sort to much documents, we can skip useless sorts
+    pub fn query(&self, query: &str, range: impl RangeBounds<usize>) -> Vec<Document> {
-    pub fn retrieve_documents(mut self, range: Range<usize>) -> Vec<Document> {
+        let mut documents = self.query_all(query);
-        let mut documents = self.retrieve_all_documents();
         let mut groups = vec![documents.as_mut_slice()];
 
         for criterion in self.criteria {
@@ -95,47 +118,65 @@ where C: Criterion
             }
         }
 
-        let range = clamp_range(range, 0..documents.len());
+        // let range = clamp_range(range, 0..documents.len());
+        let range: Range<usize> = unimplemented!();
         documents[range].to_vec()
     }
+}
 
-    pub fn retrieve_distinct_documents<K>(mut self, range: Range<usize>) -> Vec<Document>
+pub struct DistinctQueryBuilder<T: Deref<Target=DB>, F, C> {
-    where F: Fn(&DocumentId) -> Option<K>,
+    snapshot: Snapshot<T>,
-          K: Hash + Eq,
+    blob: PositiveBlob,
-    {
+    criteria: Vec<C>,
-        let mut documents = self.retrieve_all_documents();
+    function: F,
-        let mut groups = vec![documents.as_mut_slice()];
+    size: usize,
+}
 
-        for criterion in self.criteria {
+// pub struct Schema;
-            let tmp_groups = mem::replace(&mut groups, Vec::new());
+// pub struct DocDatabase;
+// where F: Fn(&Schema, &DocDatabase) -> Option<K>,
+//       K: Hash + Eq,
 
-            for group in tmp_groups {
+impl<T: Deref<Target=DB>, F, C> DistinctQueryBuilder<T, F, C>
-                group.sort_unstable_by(|a, b| criterion.evaluate(a, b));
+where T: Deref<Target=DB>,
-                for group in GroupByMut::new(group, |a, b| criterion.eq(a, b)) {
+      C: Criterion,
-                    groups.push(group);
+{
-                }
+    pub fn query(&self, query: &str, range: impl RangeBounds<usize>) -> Vec<Document> {
-            }
+        // let mut documents = self.retrieve_all_documents();
-        }
+        // let mut groups = vec![documents.as_mut_slice()];
 
-        let mut out_documents = Vec::with_capacity(range.len());
+        // for criterion in self.criteria {
-        let (distinct, limit) = self.distinct;
+        //     let tmp_groups = mem::replace(&mut groups, Vec::new());
-        let mut seen = DistinctMap::new(limit);
 
-        for document in documents {
+        //     for group in tmp_groups {
-            let accepted = match distinct(&document.id) {
+        //         group.sort_unstable_by(|a, b| criterion.evaluate(a, b));
-                Some(key) => seen.digest(key),
+        //         for group in GroupByMut::new(group, |a, b| criterion.eq(a, b)) {
-                None => seen.accept_without_key(),
+        //             groups.push(group);
-            };
+        //         }
+        //     }
+        // }
 
-            if accepted {
+        // let mut out_documents = Vec::with_capacity(range.len());
-                if seen.len() == range.end { break }
+        // let (distinct, limit) = self.distinct;
-                if seen.len() >= range.start {
+        // let mut seen = DistinctMap::new(limit);
-                    out_documents.push(document);
-                }
-            }
-        }
 
-        out_documents
+        // for document in documents {
+        //     let accepted = match distinct(&document.id) {
+        //         Some(key) => seen.digest(key),
+        //         None => seen.accept_without_key(),
+        //     };
+
+        //     if accepted {
+        //         if seen.len() == range.end { break }
+        //         if seen.len() >= range.start {
+        //             out_documents.push(document);
+        //         }
+        //     }
+        // }
+
+        // out_documents
+
+        unimplemented!()
     }
 }