From c371fb56b7b87fdf8f499cad571aa2220b2bd084 Mon Sep 17 00:00:00 2001
From: Kerollmops <renault.cle@gmail.com>
Date: Sun, 22 Apr 2018 18:10:01 +0200
Subject: [PATCH] map: Filter words a little bit

---
 src/bin/raptor-cli.rs | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/src/bin/raptor-cli.rs b/src/bin/raptor-cli.rs
index c54cfe6d0..fb1cea63e 100644
--- a/src/bin/raptor-cli.rs
+++ b/src/bin/raptor-cli.rs
@@ -6,6 +6,7 @@ extern crate raptor;
 extern crate serde_json;
 #[macro_use] extern crate serde_derive;
 
+use std::collections::HashSet;
 use std::fs::File;
 use std::io::{BufReader, BufRead};
 
@@ -25,16 +26,34 @@ fn main() {
     let data = File::open("products.json_lines").unwrap();
     let data = BufReader::new(data);
 
+    let common_words = {
+        // TODO don't break if doesn't exist
+        let file = File::open("fr.stopwords.txt").unwrap();
+        let file = BufReader::new(file);
+        let mut set = HashSet::new();
+
+        for line in file.lines() {
+            let words = line.unwrap();
+            for word in words.split_whitespace() {
+                set.insert(word.to_owned());
+            }
+        }
+
+        set
+    };
+
     let mut builder = MultiMapBuilder::new();
     for line in data.lines() {
         let line = line.unwrap();
 
+        // TODO if possible remove String allocation of Product here...
         let product: Product = from_str(&line).unwrap();
 
-        // TODO filter words here !!!
         let title = product.title.split_whitespace();
-        let description = product.ft.split_whitespace();
-        let words = title.chain(description);
+        let description = product.ft.split_whitespace().filter(|&s| s != "Description");
+        let words = title.chain(description)
+                         .filter(|&s| s.chars().any(|c| c.is_alphabetic())) // remove that ?
+                         .filter(|&s| !common_words.contains(s));
 
         for word in words {
             builder.insert(word, product.product_id);
@@ -47,8 +66,8 @@ fn main() {
 
     let map = unsafe { MultiMap::from_paths("map.fst", "values.vecs").unwrap() };
 
-    let mut stream = map.stream();
-    while let Some(x) = stream.next() {
-        println!("{:?}", x);
-    }
+    // let mut stream = map.stream();
+    // while let Some(x) = stream.next() {
+    //     println!("{:?}", x);
+    // }
 }