From 23134fee024807649bb19a7384c31e33780ede12 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 25 Jun 2018 22:26:49 +0200 Subject: [PATCH] feat: Fix the ranking algorithm to sort only the needed documents --- Cargo.lock | 16 +-- raptor-indexer/Cargo.lock | 22 +-- raptor-indexer/src/main.rs | 26 +++- raptor-search/Cargo.lock | 26 ++-- raptor-search/src/main.rs | 6 +- src/map.rs | 2 +- src/rank.rs | 288 +++++++++++++------------------------ 7 files changed, 156 insertions(+), 230 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19ce15488..7eb377a60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,7 +4,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -24,7 +24,7 @@ dependencies = [ [[package]] name = "group-by" version = "0.1.0" -source = "git+https://github.com/Kerollmops/group-by.git#7e432aa232834b650ca85ecd46056a43a0094dec" +source = "git+https://github.com/Kerollmops/group-by.git#034fadc462dc511ed53f44f6091f8707a27ca392" [[package]] name = "levenshtein_automata" @@ -72,18 +72,18 @@ dependencies = [ "fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)", "group-by 0.1.0 (git+https://github.com/Kerollmops/group-by.git)", "levenshtein_automata 0.1.1 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde_derive" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -135,8 +135,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "effdb53b25cdad54f8f48843d67398f7ef2e14f12c1b4cb4effc549a6462a4d6" "checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035" -"checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95" -"checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79" +"checksum serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "429fcc4efa8a11341b5422c2ace724daba276c1748467e869478f53c0ba4562e" +"checksum serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "6a25ad0bf818ed2d180c89addbe29198d1de6c89ed08a48aa6a4d3d16a63cbfe" "checksum syn 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c67da57e61ebc7b7b6fff56bb34440ca3a83db037320b0507af4c10368deda7d" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" diff --git a/raptor-indexer/Cargo.lock b/raptor-indexer/Cargo.lock index d99457e85..09a315df4 100644 --- a/raptor-indexer/Cargo.lock +++ b/raptor-indexer/Cargo.lock @@ -4,7 +4,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -29,7 +29,7 @@ dependencies = [ [[package]] name = "group-by" version = "0.1.0" -source = "git+https://github.com/Kerollmops/group-by.git#7e432aa232834b650ca85ecd46056a43a0094dec" +source = "git+https://github.com/Kerollmops/group-by.git#034fadc462dc511ed53f44f6091f8707a27ca392" [[package]] name = "itoa" @@ -82,8 +82,8 @@ dependencies = [ "fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)", "group-by 0.1.0 (git+https://github.com/Kerollmops/group-by.git)", "levenshtein_automata 0.1.1 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -91,20 +91,20 @@ name = "raptor-indexer" version = "0.1.0" dependencies = [ "raptor 0.1.0", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)", "unidecode 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde_derive" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -119,7 +119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -173,8 +173,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "effdb53b25cdad54f8f48843d67398f7ef2e14f12c1b4cb4effc549a6462a4d6" "checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035" -"checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95" -"checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79" +"checksum serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "429fcc4efa8a11341b5422c2ace724daba276c1748467e869478f53c0ba4562e" +"checksum serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "6a25ad0bf818ed2d180c89addbe29198d1de6c89ed08a48aa6a4d3d16a63cbfe" "checksum serde_json 1.0.22 (registry+https://github.com/rust-lang/crates.io-index)" = "84b8035cabe9b35878adec8ac5fe03d5f6bc97ff6edd7ccb96b44c1276ba390e" "checksum syn 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c67da57e61ebc7b7b6fff56bb34440ca3a83db037320b0507af4c10368deda7d" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" diff --git a/raptor-indexer/src/main.rs b/raptor-indexer/src/main.rs index 4e1076226..2a2aec211 100644 --- a/raptor-indexer/src/main.rs +++ b/raptor-indexer/src/main.rs @@ -31,6 +31,12 @@ where P: AsRef fs::set_permissions(&path, perms) } +fn is_readonly

(path: P) -> io::Result +where P: AsRef +{ + fs::metadata(&path).map(|m| m.permissions().readonly()) +} + fn main() { let data = File::open("products.json_lines").unwrap(); let data = BufReader::new(data); @@ -54,6 +60,18 @@ fn main() { } }; + let map_file = "map.fst"; + let values_file = "values.vecs"; + + for file in &[map_file, values_file] { + match is_readonly(file) { + Ok(true) => panic!("the {:?} file is readonly, please make it writeable", file), + Err(ref e) if e.kind() == io::ErrorKind::NotFound => (), + Err(e) => panic!("{:?}", e), + _ => (), + } + } + let mut builder = DocIndexMapBuilder::new(); for line in data.lines() { let line = line.unwrap(); @@ -83,13 +101,13 @@ fn main() { } } - let map = File::create("map.fst").unwrap(); - let values = File::create("values.vecs").unwrap(); + let map = File::create(map_file).unwrap(); + let values = File::create(values_file).unwrap(); let (map, values) = builder.build(map, values).unwrap(); - set_readonly("map.fst", true).unwrap(); - set_readonly("values.vecs", true).unwrap(); + set_readonly(map_file, true).unwrap(); + set_readonly(values_file, true).unwrap(); println!("Checking the dump consistency..."); unsafe { DocIndexMap::from_paths("map.fst", "values.vecs").unwrap() }; diff --git a/raptor-search/Cargo.lock b/raptor-search/Cargo.lock index a008303b7..25cd776dd 100644 --- a/raptor-search/Cargo.lock +++ b/raptor-search/Cargo.lock @@ -4,7 +4,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -14,7 +14,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "cfg-if" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -42,7 +42,7 @@ dependencies = [ [[package]] name = "group-by" version = "0.1.0" -source = "git+https://github.com/Kerollmops/group-by.git#7e432aa232834b650ca85ecd46056a43a0094dec" +source = "git+https://github.com/Kerollmops/group-by.git#034fadc462dc511ed53f44f6091f8707a27ca392" [[package]] name = "levenshtein_automata" @@ -70,7 +70,7 @@ name = "log" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -106,8 +106,8 @@ dependencies = [ "fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)", "group-by 0.1.0 (git+https://github.com/Kerollmops/group-by.git)", "levenshtein_automata 0.1.1 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -118,18 +118,18 @@ dependencies = [ "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)", "raptor 0.1.0", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde_derive" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -174,7 +174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] "checksum bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9f2fb9e29e72fd6bc12071533d5dc7664cb01480c59406f656d7ac25c7bd8ff7" "checksum byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "74c0b906e9446b0a2e4f760cdb3fa4b2c48cdc6db8766a845c54b6ff063fd2e9" -"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" +"checksum cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efe5c877e17a9c717a0bf3613b2709f723202c4e4675cc8f12926ded29bcb17e" "checksum elapsed 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f4e5af126dafd0741c2ad62d47f68b28602550102e5f0dd45c8a97fc8b49c29" "checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f" "checksum fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)" = "" @@ -186,8 +186,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "effdb53b25cdad54f8f48843d67398f7ef2e14f12c1b4cb4effc549a6462a4d6" "checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035" -"checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95" -"checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79" +"checksum serde 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "429fcc4efa8a11341b5422c2ace724daba276c1748467e869478f53c0ba4562e" +"checksum serde_derive 1.0.68 (registry+https://github.com/rust-lang/crates.io-index)" = "6a25ad0bf818ed2d180c89addbe29198d1de6c89ed08a48aa6a4d3d16a63cbfe" "checksum syn 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c67da57e61ebc7b7b6fff56bb34440ca3a83db037320b0507af4c10368deda7d" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" diff --git a/raptor-search/src/main.rs b/raptor-search/src/main.rs index 503571599..0b156719b 100644 --- a/raptor-search/src/main.rs +++ b/raptor-search/src/main.rs @@ -3,8 +3,7 @@ extern crate fst; extern crate raptor; extern crate elapsed; -use std::{env, fs}; -use std::process::Command; +use std::env; use std::io::{self, Write}; use elapsed::measure_time; use fst::Streamer; @@ -21,7 +20,8 @@ fn search(map: &DocIndexMap, lev_builder: &LevBuilder, query: &str) { while let Some(document_id) = stream.next() { print!("{:?}", document_id); - /* only here to debug ! + // /* only here to debug ! + use std::{fs, process::Command}; if let Ok(_) = fs::File::open("products.json_lines") { let output = Command::new("rg") .arg(document_id.to_string()) diff --git a/src/map.rs b/src/map.rs index 9b32918e9..bd27ede7e 100644 --- a/src/map.rs +++ b/src/map.rs @@ -86,7 +86,7 @@ impl Values { let cap = raw.iter().map(Vec::len).sum(); let mut values = Vec::with_capacity(cap); - for v in &raw { + for mut v in &raw { let len = v.len() as u64; let start = ranges.last().map(|&Range { end, .. }| end).unwrap_or(0); diff --git a/src/rank.rs b/src/rank.rs index 567033072..c73836e0e 100644 --- a/src/rank.rs +++ b/src/rank.rs @@ -1,6 +1,6 @@ use std::cmp::{self, Ordering}; -use std::{mem, vec}; -use std::collections::{HashSet, HashMap}; +use std::collections::HashMap; +use std::{mem, vec, iter}; use DocIndexMap; use fst; use levenshtein_automata::DFA; @@ -10,11 +10,16 @@ use map::{ Values, }; use {Match, DocIndex, DocumentId}; -use group_by::GroupBy; +use group_by::{GroupBy, GroupByMut}; -const MAX_DISTANCE: usize = 8; +const MAX_DISTANCE: u32 = 8; -#[derive(Debug, Eq, Clone)] +#[inline] +fn match_query_index(a: &Match, b: &Match) -> bool { + a.query_index == b.query_index +} + +#[derive(Debug, Clone)] pub struct Document { document_id: DocumentId, matches: Vec, @@ -33,172 +38,83 @@ impl Document { } } -impl PartialEq for Document { - fn eq(&self, other: &Self) -> bool { - self.cmp(other) == Ordering::Equal +fn sum_of_typos(lhs: &Document, rhs: &Document) -> Ordering { + fn sum_of_typos(doc: &Document) -> u8 { + GroupBy::new(&doc.matches, match_query_index).map(|m| m[0].distance).sum() } + sum_of_typos(lhs).cmp(&sum_of_typos(rhs)) } -impl PartialOrd for Document { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) +fn number_of_words(lhs: &Document, rhs: &Document) -> Ordering { + fn number_of_words(doc: &Document) -> usize { + GroupBy::new(&doc.matches, match_query_index).count() } + number_of_words(lhs).cmp(&number_of_words(rhs)).reverse() } -impl Ord for Document { - fn cmp(&self, other: &Self) -> Ordering { - let lhs = DocumentScore::new(&self.matches); - let rhs = DocumentScore::new(&other.matches); - lhs.cmp(&rhs) - } -} - -#[derive(Debug, Default, Eq, PartialEq, PartialOrd)] -struct DocumentScore { - typo: usize, - words: usize, - proximity: usize, - attribute: usize, - words_position: usize, -} - -impl Ord for DocumentScore { - fn cmp(&self, other: &Self) -> Ordering { - self.typo.cmp(&other.typo) - .then(self.words.cmp(&other.words).reverse()) - .then(self.proximity.cmp(&other.proximity)) - .then(self.attribute.cmp(&other.attribute)) - .then(self.words_position.cmp(&other.words_position)) - // ~exact~ (see prefix option of the `DFA` builder) - } -} - -fn min_attribute(matches: &[Match]) -> usize { - let mut attribute = usize::max_value(); - for match_ in matches { - if match_.attribute == 0 { return 0 } - attribute = cmp::min(match_.attribute as usize, attribute); - } - attribute -} - -fn min_attribute_index(matches: &[Match]) -> usize { - let mut attribute_index = usize::max_value(); - for match_ in matches { - if match_.attribute_index == 0 { return 0 } - attribute_index = cmp::min(match_.attribute_index as usize, attribute_index); - } - attribute_index -} - -impl DocumentScore { - fn new(matches: &[Match]) -> Self { - let mut score = DocumentScore::default(); - - let mut index = 0; // FIXME could be replaced by the `GroupBy::remaining` method - for group in GroupBy::new(matches, |a, b| a.query_index == b.query_index) { - index += group.len(); - - score.typo = cmp::max(group[0].distance as usize, score.typo); - score.words += 1; - - // FIXME distance is wrong if 2 different attributes matches - if let Some(first_next_group) = (&matches[index..]).first() { - score.proximity += attribute_proximity(first_next_group, &group[0]); +fn words_proximity(lhs: &Document, rhs: &Document) -> Ordering { + fn word_proximity(doc: &Document) -> u32 { + fn attribute_proximity(lhs: &Match, rhs: &Match) -> u32 { + fn index_proximity(lhs: u32, rhs: u32) -> u32 { + if lhs < rhs { + cmp::min(rhs - lhs, MAX_DISTANCE) + } else { + cmp::min(lhs - rhs, MAX_DISTANCE) + 1 + } } - score.attribute += min_attribute(group); - score.words_position += min_attribute_index(group); + if lhs.attribute != rhs.attribute { return MAX_DISTANCE } + index_proximity(lhs.attribute_index, rhs.attribute_index) } - score + let mut proximity = 0; + let mut next_group_index = 0; + for group in GroupBy::new(&doc.matches, match_query_index) { + next_group_index += group.len(); + // FIXME distance is wrong if 2 different attributes matches + // FIXME do that in a manner to avoid memory cache misses + if let Some(first_next_group) = doc.matches.get(next_group_index) { + proximity += attribute_proximity(first_next_group, &group[0]); + } + } + proximity } + word_proximity(lhs).cmp(&word_proximity(rhs)) } -fn proximity(first: usize, second: usize) -> usize { - if first < second { - cmp::min(second - first, MAX_DISTANCE) - } else { - cmp::min(first - second, MAX_DISTANCE) + 1 +fn sum_of_words_attribute(lhs: &Document, rhs: &Document) -> Ordering { + fn sum_attribute(doc: &Document) -> u8 { + GroupBy::new(&doc.matches, match_query_index).map(|m| m[0].attribute).sum() } + sum_attribute(lhs).cmp(&sum_attribute(rhs)) } -fn attribute_proximity(lhs: &Match, rhs: &Match) -> usize { - if lhs.attribute != rhs.attribute { - MAX_DISTANCE - } else { - let lhs_attr = lhs.attribute_index as usize; - let rhs_attr = rhs.attribute_index as usize; - proximity(lhs_attr, rhs_attr) +fn sum_of_words_position(lhs: &Document, rhs: &Document) -> Ordering { + fn sum_attribute_index(doc: &Document) -> u32 { + GroupBy::new(&doc.matches, match_query_index).map(|m| m[0].attribute_index).sum() } + sum_attribute_index(lhs).cmp(&sum_attribute_index(rhs)) +} + +fn exact(lhs: &Document, rhs: &Document) -> Ordering { + unimplemented!() } pub struct Pool { - returned_documents: HashSet, documents: Vec, - limitation: Limitation, -} - -#[derive(Debug, Copy, Clone)] -enum Limitation { - /// No limitation is specified. - Unspecified { // FIXME rename that ! - /// The maximum number of results to return. - limit: usize, - }, - - /// The limitation is specified but not reached. - Specified { - /// The maximum number of results to return. - limit: usize, - - /// documents with a distance of zero which can be used - /// in the step-by-step sort-and-return. - /// - /// this field must be equal to the limit to reach - /// the limitation - matching_documents: usize, - }, - - /// No more documents with a distance of zero - /// can never be returned now. - Reached { - /// The number of remaining documents to return in order. - remaining: usize, - }, -} - -impl Limitation { - fn reached(&self) -> Option { - match self { - Limitation::Reached { remaining } => Some(*remaining), - _ => None, - } - } - - fn is_reached(&self) -> bool { - self.reached().is_some() - } + limit: usize, } impl Pool { pub fn new(query_size: usize, limit: usize) -> Self { - assert!(query_size > 0, "query size can not be less that one"); - assert!(limit > 0, "limit can not be less that one"); - - let limitation = match query_size { - 1 => Limitation::Specified { limit, matching_documents: 0 }, - _ => Limitation::Unspecified { limit }, - }; - Self { - returned_documents: HashSet::new(), documents: Vec::new(), - limitation: limitation, + limit: limit, } } - pub fn extend(&mut self, mut matches: HashMap>) { + // TODO remove the matches HashMap, not proud of it + pub fn extend(&mut self, matches: &mut HashMap>) { for doc in self.documents.iter_mut() { if let Some(matches) = matches.remove(&doc.document_id) { doc.matches.extend(matches); @@ -206,65 +122,48 @@ impl Pool { } } - matches.retain(|id, _| !self.returned_documents.contains(id)); - self.documents.reserve(matches.len()); - - let mut new_matches = 0; - for (id, mut matches) in matches.into_iter() { + for (id, mut matches) in matches.drain() { + // note that matches are already sorted we do that by security + // TODO remove this useless sort matches.sort_unstable(); - if matches[0].distance == 0 { new_matches += 1 } - - if self.limitation.is_reached() { - match matches.iter().position(|match_| match_.distance > 0) { - Some(pos) if pos == 0 => continue, - Some(pos) => matches.truncate(pos), - None => (), - } - } let document = Document::from_sorted_matches(id, matches); self.documents.push(document); } - self.documents.sort_unstable(); - - self.limitation = match self.limitation { - Limitation::Specified { limit, matching_documents } if matching_documents + new_matches >= limit => { - // this is the biggest valid match - // used to find the next smallest invalid match - let biggest_valid = Match { query_index: 0, distance: 0, ..Match::max() }; - - // documents which does not have a match with a distance of 0 can be removed. - // note that documents have a query size of 1. - match self.documents.binary_search_by(|d| d.matches[0].cmp(&biggest_valid)) { - Ok(index) => self.documents.truncate(index + 1), // this will never happen :) - Err(index) => self.documents.truncate(index), - } - - Limitation::Reached { remaining: limit } - }, - Limitation::Specified { limit, matching_documents } => { - Limitation::Specified { - limit: limit, - matching_documents: matching_documents + new_matches - } - }, - limitation => limitation, - }; } } +fn invert_sorts(a: &Document, b: &Document, sorts: &[F]) -> bool +where F: Fn(&Document, &Document) -> Ordering, +{ + sorts.iter().rev().all(|sort| sort(a, b) == Ordering::Equal) +} + impl IntoIterator for Pool { type Item = Document; type IntoIter = vec::IntoIter; fn into_iter(mut self) -> Self::IntoIter { - let limit = match self.limitation { - Limitation::Unspecified { limit } => limit, - Limitation::Specified { limit, .. } => limit, - Limitation::Reached { remaining } => remaining, - }; + let sorts = &[ + sum_of_typos, + number_of_words, + words_proximity, + sum_of_words_attribute, + sum_of_words_position, + ]; - self.documents.truncate(limit); + for (i, sort) in sorts.iter().enumerate() { + let mut computed = 0; + for group in GroupByMut::new(&mut self.documents, |a, b| invert_sorts(a, b, &sorts[..i])) { + // TODO prefer using `sort_unstable_by_key` to allow reusing the key computation + // `number of words` needs to be reversed, we can use the `cmp::Reverse` struct to do that + group.sort_unstable_by(sort); + computed += group.len(); + if computed >= self.limit { break } + } + } + + self.documents.truncate(self.limit); self.documents.into_iter() } } @@ -303,6 +202,8 @@ impl<'m, 'v, 'a> fst::Streamer<'a> for RankedStream<'m, 'v> { type Item = DocumentId; fn next(&'a mut self) -> Option { + let mut matches = HashMap::new(); + loop { // TODO remove that when NLL are here ! let mut transfert_pool = None; @@ -313,14 +214,21 @@ impl<'m, 'v, 'a> fst::Streamer<'a> for RankedStream<'m, 'v> { Some((_string, indexed_values)) => { for iv in indexed_values { + // TODO extend documents matches by batch of query_index + // that way it will be possible to discard matches that + // have an invalid distance *before* adding them + // to the matches of the documents and, that way, avoid a sort + + // let string = unsafe { str::from_utf8_unchecked(_string) }; + // println!("for {:15} ", string); + let distance = automatons[iv.index].distance(iv.state).to_u8(); // TODO remove the Pool system ! // this is an internal Pool rule but // it is more efficient to test that here - if pool.limitation.is_reached() && distance != 0 { continue } + // if pool.limitation.is_reached() && distance != 0 { continue } - let mut matches = HashMap::with_capacity(iv.values.len() / 2); for di in iv.values { let match_ = Match { query_index: iv.index as u32, @@ -329,10 +237,10 @@ impl<'m, 'v, 'a> fst::Streamer<'a> for RankedStream<'m, 'v> { attribute_index: di.attribute_index, }; matches.entry(di.document) - .and_modify(|matches: &mut Vec<_>| matches.push(match_)) + .and_modify(|ms: &mut Vec<_>| ms.push(match_)) .or_insert_with(|| vec![match_]); } - pool.extend(matches); + pool.extend(&mut matches); } }, None => {