feat: Define a DocIndex struct

raptor-bin/src/main.rs

@@ -9,7 +9,7 @@ use std::fs::File;
 use std::io::{BufReader, BufRead};
 use std::iter;
 
-use raptor::{MapBuilder, Map, Value, AttrIndex};
+use raptor::{DocIndexMapBuilder, DocIndexMap, DocIndex};
 use serde_json::from_str;
 
 fn main() {
@@ -35,7 +35,7 @@ fn main() {
         }
     };
 
-    let mut builder = MapBuilder::new();
+    let mut builder = DocIndexMapBuilder::new();
     for line in data.lines() {
         let line = line.unwrap();
 
@@ -51,14 +51,12 @@ fn main() {
 
         let words = title.chain(description);
         for (i, (attr, word)) in words {
-            let value = Value {
+            let doc_index = DocIndex {
-                id: product["product_id"].as_u64().expect("invalid `product_id`"),
+                document: product["product_id"].as_u64().expect("invalid `product_id`"),
-                attr_index: AttrIndex {
                 attribute: attr,
-                    index: i as u64,
+                attribute_index: i as u32,
-                },
             };
-            builder.insert(word, value);
+            builder.insert(word, doc_index);
         }
     }
 
@@ -66,6 +64,6 @@ fn main() {
     let values = File::create("values.vecs").unwrap();
     let (map, values) = builder.build(map, values).unwrap();
 
-    eprintln!("Checking the dump consistency...");
+    println!("Checking the dump consistency...");
-    unsafe { Map::<Value>::from_paths("map.fst", "values.vecs").unwrap() };
+    unsafe { DocIndexMap::from_paths("map.fst", "values.vecs").unwrap() };
 }

raptor-http/src/main.rs

@@ -16,10 +16,10 @@ use tokio_minihttp::{Request, Response, Http};
 use tokio_proto::TcpServer;
 use tokio_service::Service;
 
-use raptor::{Map, OpWithStateBuilder, LevBuilder, Value};
+use raptor::{DocIndexMap, OpWithStateBuilder, LevBuilder};
 
 struct MainService {
-    map: Arc<Map<Value>>,
+    map: Arc<DocIndexMap>,
     lev_builder: Arc<LevBuilder>,
 }
 
@@ -92,7 +92,7 @@ fn main() {
     let map = {
         let fst = fs::read("map.fst").unwrap();
         let values = fs::read("values.vecs").unwrap();
-        let map = Map::from_bytes(fst, &values).unwrap();
+        let map = DocIndexMap::from_bytes(fst, &values).unwrap();
         Arc::new(map)
     };
 

src/lib.rs

@@ -16,14 +16,64 @@ pub use self::map::{
 pub use self::capped_btree_map::{CappedBTreeMap, Insertion};
 pub use self::levenshtein::LevBuilder;
 
-#[derive(Debug, Serialize, Deserialize)]
+pub type DocIndexMap = Map<DocIndex>;
-pub struct Value {
+pub type DocIndexMapBuilder = MapBuilder<DocIndex>;
-    pub id: u64,
+
-    pub attr_index: AttrIndex,
+/// This structure represent the position of a word
+/// in a document and its attributes.
+///
+/// This is stored in the map, generated at index time,
+/// extracted and interpreted at search time.
+#[derive(Debug, Copy, Clone, Eq, PartialEq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct DocIndex {
+
+    /// The document identifier where the word was found.
+    pub document: u64,
+
+    /// The attribute identifier in the document
+    /// where the word was found.
+    ///
+    /// This is an `u8` therefore a document
+    /// can not have more than `2^8` attributes.
+    pub attribute: u8,
+
+    /// The index where the word was found in the attribute.
+    ///
+    /// Only the first 1000 words are indexed.
+    pub attribute_index: u32,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+/// This structure represent a matching word with informations
-pub struct AttrIndex {
+/// on the location of the word in the document.
+///
+/// The order of the field is important because it defines
+/// the way these structures are ordered between themselves.
+///
+/// The word in itself is not important.
+#[derive(Debug, Copy, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)]
+pub struct Match {
+
+    /// The distance the word has with the query word
+    /// (i.e. the Levenshtein distance).
+    pub distance: u8,
+
+    /// The attribute in which the word is located
+    /// (i.e. Title is 0, Description is 1).
+    ///
+    /// This is an `u8` therefore a document
+    /// can not have more than `2^8` attributes.
     pub attribute: u8,
-    pub index: u64,
+
+    /// The word index in the query sentence.
+    /// Same as the `attribute_index` but for the query words.
+    ///
+    /// Used to retrieve the automaton that match this word.
+    pub query_index: u32,
+
+    /// Where does this word is located in the attribute string
+    /// (i.e. at the start or the end of the attribute).
+    ///
+    /// The index in the attribute is limited to a maximum of `2^32`
+    /// this is because we index only the first 1000 words in an attribute.
+    pub attribute_index: u32,
 }