feat: Remove the DatabaseView dependencies from the QueryBuilder

This commit is contained in:
Clément Renault 2019-02-24 11:58:22 +01:00
parent 58bff3d4ac
commit d664221c64
No known key found for this signature in database
GPG Key ID: 0151CDAB43460DAE
4 changed files with 39 additions and 53 deletions

View File

@ -156,7 +156,7 @@ fn main() -> Result<(), Box<Error>> {
let start = Instant::now(); let start = Instant::now();
let builder = view.query_builder().unwrap(); let builder = view.query_builder();
let documents = builder.query(query, 0..opt.number_results); let documents = builder.query(query, 0..opt.number_results);
let number_of_documents = documents.len(); let number_of_documents = documents.len();

View File

@ -32,7 +32,6 @@ pub use self::schema::Schema;
pub use self::index::Index; pub use self::index::Index;
pub use self::number::{Number, ParseNumberError}; pub use self::number::{Number, ParseNumberError};
pub type RankedMap = HashMap<(DocumentId, SchemaAttr), Number>; pub type RankedMap = HashMap<(DocumentId, SchemaAttr), Number>;
const DATA_INDEX: &[u8] = b"data-index"; const DATA_INDEX: &[u8] = b"data-index";
@ -803,7 +802,7 @@ mod bench {
bench.iter(|| { bench.iter(|| {
for q in &["a", "b", "c", "d", "e"] { for q in &["a", "b", "c", "d", "e"] {
let documents = view.query_builder().unwrap().query(q, 0..20); let documents = view.query_builder().query(q, 0..20);
test::black_box(|| documents); test::black_box(|| documents);
} }
}); });
@ -851,7 +850,7 @@ mod bench {
bench.iter(|| { bench.iter(|| {
for q in &["a", "b", "c", "d", "e"] { for q in &["a", "b", "c", "d", "e"] {
let documents = view.query_builder().unwrap().query(q, 0..20); let documents = view.query_builder().query(q, 0..20);
test::black_box(|| documents); test::black_box(|| documents);
} }
}); });
@ -900,7 +899,7 @@ mod bench {
bench.iter(|| { bench.iter(|| {
for q in &["a", "b", "c", "d", "e"] { for q in &["a", "b", "c", "d", "e"] {
let documents = view.query_builder().unwrap().query(q, 0..20); let documents = view.query_builder().query(q, 0..20);
test::black_box(|| documents); test::black_box(|| documents);
} }
}); });

View File

@ -85,8 +85,8 @@ where D: Deref<Target=DB>
Ok(()) Ok(())
} }
pub fn query_builder(&self) -> Result<QueryBuilder<D, FilterFunc<D>>, Box<Error>> { pub fn query_builder(&self) -> QueryBuilder<FilterFunc> {
QueryBuilder::new(self) QueryBuilder::new(self.index())
} }
pub fn raw_field_by_document_id( pub fn raw_field_by_document_id(

View File

@ -1,7 +1,6 @@
use std::{cmp, mem, vec, str, char}; use std::{cmp, mem};
use std::ops::{Deref, Range}; use std::ops::Range;
use std::time::Instant; use std::time::Instant;
use std::error::Error;
use std::hash::Hash; use std::hash::Hash;
use std::rc::Rc; use std::rc::Rc;
@ -9,13 +8,12 @@ use rayon::slice::ParallelSliceMut;
use slice_group_by::{GroupByMut, LinearStrGroupBy}; use slice_group_by::{GroupByMut, LinearStrGroupBy};
use hashbrown::HashMap; use hashbrown::HashMap;
use fst::Streamer; use fst::Streamer;
use rocksdb::DB;
use log::info; use log::info;
use crate::automaton::{self, DfaExt, AutomatonExt}; use crate::automaton::{self, DfaExt, AutomatonExt};
use crate::rank::distinct_map::{DistinctMap, BufferedDistinctMap}; use crate::rank::distinct_map::{DistinctMap, BufferedDistinctMap};
use crate::rank::criterion::Criteria; use crate::rank::criterion::Criteria;
use crate::database::DatabaseView; use crate::database::Index;
use crate::rank::{raw_documents_from_matches, RawDocument, Document}; use crate::rank::{raw_documents_from_matches, RawDocument, Document};
use crate::{is_cjk, Match, DocumentId}; use crate::{is_cjk, Match, DocumentId};
@ -63,43 +61,38 @@ fn split_whitespace_automatons(query: &str) -> Vec<DfaExt> {
automatons automatons
} }
pub type FilterFunc<D> = fn(DocumentId, &DatabaseView<D>) -> bool; pub type FilterFunc = fn(DocumentId) -> bool;
pub struct QueryBuilder<'a, 'b, D, FI> pub struct QueryBuilder<'i, 'c, FI> {
where D: Deref<Target=DB> index: &'i Index,
{ criteria: Criteria<'c>,
view: &'a DatabaseView<D>,
criteria: Criteria<'b>,
filter: Option<FI>, filter: Option<FI>,
} }
impl<'a, 'b, D> QueryBuilder<'a, 'b, D, FilterFunc<D>> impl<'i, 'c> QueryBuilder<'i, 'c, FilterFunc> {
where D: Deref<Target=DB> pub fn new(index: &'i Index) -> Self {
{ QueryBuilder::with_criteria(index, Criteria::default())
pub fn new(view: &'a DatabaseView<D>) -> Result<Self, Box<Error>> {
QueryBuilder::with_criteria(view, Criteria::default())
} }
pub fn with_criteria(view: &'a DatabaseView<D>, criteria: Criteria<'b>) -> Result<Self, Box<Error>> { pub fn with_criteria(index: &'i Index, criteria: Criteria<'c>) -> Self {
Ok(QueryBuilder { view, criteria, filter: None }) QueryBuilder { index, criteria, filter: None }
} }
} }
impl<'a, 'b, D, FI> QueryBuilder<'a, 'b, D, FI> impl<'i, 'c, FI> QueryBuilder<'i, 'c, FI>
where D: Deref<Target=DB>,
{ {
pub fn with_filter<F>(self, function: F) -> QueryBuilder<'a, 'b, D, F> pub fn with_filter<F>(self, function: F) -> QueryBuilder<'i, 'c, F>
where F: Fn(DocumentId, &DatabaseView<D>) -> bool, where F: Fn(DocumentId) -> bool,
{ {
QueryBuilder { QueryBuilder {
view: self.view, index: self.index,
criteria: self.criteria, criteria: self.criteria,
filter: Some(function) filter: Some(function)
} }
} }
pub fn with_distinct<F, K>(self, function: F, size: usize) -> DistinctQueryBuilder<'a, 'b, D, FI, F> pub fn with_distinct<F, K>(self, function: F, size: usize) -> DistinctQueryBuilder<'i, 'c, FI, F>
where F: Fn(DocumentId, &DatabaseView<D>) -> Option<K>, where F: Fn(DocumentId) -> Option<K>,
K: Hash + Eq, K: Hash + Eq,
{ {
DistinctQueryBuilder { DistinctQueryBuilder {
@ -115,7 +108,7 @@ where D: Deref<Target=DB>,
let mut stream = { let mut stream = {
let mut op_builder = fst::map::OpBuilder::new(); let mut op_builder = fst::map::OpBuilder::new();
for automaton in &automatons { for automaton in &automatons {
let stream = self.view.index().map.search(automaton); let stream = self.index.map.search(automaton);
op_builder.push(stream); op_builder.push(stream);
} }
op_builder.union() op_builder.union()
@ -129,7 +122,7 @@ where D: Deref<Target=DB>,
let distance = automaton.eval(input).to_u8(); let distance = automaton.eval(input).to_u8();
let is_exact = distance == 0 && input.len() == automaton.query_len(); let is_exact = distance == 0 && input.len() == automaton.query_len();
let doc_indexes = &self.view.index().indexes; let doc_indexes = &self.index.indexes;
let doc_indexes = &doc_indexes[iv.value as usize]; let doc_indexes = &doc_indexes[iv.value as usize];
for doc_index in doc_indexes { for doc_index in doc_indexes {
@ -157,15 +150,14 @@ where D: Deref<Target=DB>,
} }
} }
impl<'a, 'b, D, FI> QueryBuilder<'a, 'b, D, FI> impl<'i, 'c, FI> QueryBuilder<'i, 'c, FI>
where D: Deref<Target=DB>, where FI: Fn(DocumentId) -> bool,
FI: Fn(DocumentId, &DatabaseView<D>) -> bool,
{ {
pub fn query(self, query: &str, range: Range<usize>) -> Vec<Document> { pub fn query(self, query: &str, range: Range<usize>) -> Vec<Document> {
// We delegate the filter work to the distinct query builder, // We delegate the filter work to the distinct query builder,
// specifying a distinct rule that has no effect. // specifying a distinct rule that has no effect.
if self.filter.is_some() { if self.filter.is_some() {
let builder = self.with_distinct(|_, _| None as Option<()>, 1); let builder = self.with_distinct(|_| None as Option<()>, 1);
return builder.query(query, range); return builder.query(query, range);
} }
@ -211,19 +203,16 @@ where D: Deref<Target=DB>,
} }
} }
pub struct DistinctQueryBuilder<'a, 'b, D, FI, FD> pub struct DistinctQueryBuilder<'i, 'c, FI, FD> {
where D: Deref<Target=DB> inner: QueryBuilder<'i, 'c, FI>,
{
inner: QueryBuilder<'a, 'b, D, FI>,
function: FD, function: FD,
size: usize, size: usize,
} }
impl<'a, 'b, D, FI, FD> DistinctQueryBuilder<'a, 'b, D, FI, FD> impl<'i, 'c, FI, FD> DistinctQueryBuilder<'i, 'c, FI, FD>
where D: Deref<Target=DB>,
{ {
pub fn with_filter<F>(self, function: F) -> DistinctQueryBuilder<'a, 'b, D, F, FD> pub fn with_filter<F>(self, function: F) -> DistinctQueryBuilder<'i, 'c, F, FD>
where F: Fn(DocumentId, &DatabaseView<D>) -> bool, where F: Fn(DocumentId) -> bool,
{ {
DistinctQueryBuilder { DistinctQueryBuilder {
inner: self.inner.with_filter(function), inner: self.inner.with_filter(function),
@ -233,10 +222,9 @@ where D: Deref<Target=DB>,
} }
} }
impl<'a, 'b, D, FI, FD, K> DistinctQueryBuilder<'a, 'b, D, FI, FD> impl<'i, 'c, FI, FD, K> DistinctQueryBuilder<'i, 'c, FI, FD>
where D: Deref<Target=DB>, where FI: Fn(DocumentId) -> bool,
FI: Fn(DocumentId, &DatabaseView<D>) -> bool, FD: Fn(DocumentId) -> Option<K>,
FD: Fn(DocumentId, &DatabaseView<D>) -> Option<K>,
K: Hash + Eq, K: Hash + Eq,
{ {
pub fn query(self, query: &str, range: Range<usize>) -> Vec<Document> { pub fn query(self, query: &str, range: Range<usize>) -> Vec<Document> {
@ -246,7 +234,6 @@ where D: Deref<Target=DB>,
let mut groups = vec![documents.as_mut_slice()]; let mut groups = vec![documents.as_mut_slice()];
let mut key_cache = HashMap::new(); let mut key_cache = HashMap::new();
let view = &self.inner.view;
let mut filter_map = HashMap::new(); let mut filter_map = HashMap::new();
// these two variables informs on the current distinct map and // these two variables informs on the current distinct map and
@ -281,14 +268,14 @@ where D: Deref<Target=DB>,
let filter_accepted = match &self.inner.filter { let filter_accepted = match &self.inner.filter {
Some(filter) => { Some(filter) => {
let entry = filter_map.entry(document.id); let entry = filter_map.entry(document.id);
*entry.or_insert_with(|| (filter)(document.id, view)) *entry.or_insert_with(|| (filter)(document.id))
}, },
None => true, None => true,
}; };
if filter_accepted { if filter_accepted {
let entry = key_cache.entry(document.id); let entry = key_cache.entry(document.id);
let key = entry.or_insert_with(|| (self.function)(document.id, view).map(Rc::new)); let key = entry.or_insert_with(|| (self.function)(document.id).map(Rc::new));
match key.clone() { match key.clone() {
Some(key) => buf_distinct.register(key), Some(key) => buf_distinct.register(key),