Switch to a JSON protocol for the front page

Cargo.lock

@@ -649,12 +649,9 @@ checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177"
 
 [[package]]
 name = "hashbrown"
-version = "0.8.2"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e91b62f79061a0bc2e046024cb7ba44b08419ed238ecbd9adbd787434b9e8c25"
+checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
-dependencies = [
- "autocfg 1.0.0",
-]
 
 [[package]]
 name = "headers"
@@ -811,12 +808,13 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "1.5.1"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86b45e59b16c76b11bf9738fd5d38879d3bd28ad292d7b313608becb17ae2df9"
+checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2"
 dependencies = [
  "autocfg 1.0.0",
  "hashbrown",
+ "serde",
 ]
 
 [[package]]
@@ -1014,6 +1012,7 @@ dependencies = [
  "grenad",
  "heed",
  "human_format",
+ "indexmap",
  "itertools",
  "jemallocator",
  "levenshtein_automata",

Cargo.toml

@@ -16,6 +16,7 @@ fxhash = "0.2.1"
 grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "c390cfe" }
 heed = { version = "0.8.1", default-features = false, features = ["lmdb"] }
 human_format = "1.0.3"
+indexmap = { version = "1.6.0", features = ["serde-1"] }
 jemallocator = "0.3.2"
 levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
 linked-hash-map = "0.5.3"

public/script.js

@@ -17,16 +17,15 @@ $('#search').on('input', function () {
       data: JSON.stringify({ 'query': query }),
       contentType: 'application/json',
       success: function (data, textStatus, request) {
-        let httpResults = Papa.parse(data, { delimiter: ",", header: true, skipEmptyLines: true });
         results.innerHTML = '';
 
         let timeSpent = request.getResponseHeader('Time-Ms');
-        let numberOfDocuments = httpResults.data.length;
+        let numberOfDocuments = data.length;
         count.innerHTML = `${numberOfDocuments}`;
         time.innerHTML = `${timeSpent}ms`;
         time.classList.remove('fade-in-out');
 
-        for (element of httpResults.data) {
+        for (element of data) {
           const elem = document.createElement('li');
           elem.classList.add("document");
 

src/index.rs

@@ -26,7 +26,7 @@ pub struct Index {
     /// Maps the proximity between a pair of words with all the docids where this relation appears.
     pub word_pair_proximity_docids: Database<StrStrU8Codec, CboRoaringBitmapCodec>,
     /// Maps the document id to the document as a CSV line.
-    pub documents: Database<OwnedType<BEU32>, ByteSlice>,
+    pub documents: Database<OwnedType<BEU32>, ObkvCodec>,
 }
 
 impl Index {
@@ -74,23 +74,15 @@ impl Index {
     pub fn documents<'t>(
         &self,
         rtxn: &'t heed::RoTxn,
-        iter: impl IntoIterator<Item=DocumentId>,
+        ids: impl IntoIterator<Item=DocumentId>,
-    ) -> anyhow::Result<Vec<(DocumentId, StringRecord)>>
+    ) -> anyhow::Result<Vec<(DocumentId, obkv::KvReader<'t>)>>
     {
-        let ids: Vec<_> = iter.into_iter().collect();
+        let mut documents = Vec::new();
-        let mut content = Vec::new();
 
-        for id in ids.iter().cloned() {
+        for id in ids {
-            let document_content = self.documents.get(rtxn, &BEU32::new(id))?
+            let kv = self.documents.get(rtxn, &BEU32::new(id))?
                 .with_context(|| format!("Could not find document {}", id))?;
-            content.extend_from_slice(document_content);
+            documents.push((id, kv));
-        }
-
-        let mut rdr = csv::ReaderBuilder::new().has_headers(false).from_reader(&content[..]);
-
-        let mut documents = Vec::with_capacity(ids.len());
-        for (id, result) in ids.into_iter().zip(rdr.records()) {
-            documents.push((id, result?));
         }
 
         Ok(documents)

src/indexing/store.rs

@@ -1,5 +1,5 @@
 use std::collections::{BTreeMap, HashMap};
-use std::convert::TryFrom;
+use std::convert::{TryFrom, TryInto};
 use std::fs::File;
 use std::io::Read;
 use std::iter::FromIterator;
@@ -204,11 +204,15 @@ impl Store {
             self.insert_word_docid(word, document_id)?;
         }
 
-        let record = CsvStringRecordCodec::bytes_encode(record)
+        let mut writer = obkv::KvWriter::memory();
-            .with_context(|| format!("could not encode CSV record"))?;
+        record.iter().enumerate().for_each(|(i, v)| {
+            let key = i.try_into().unwrap();
+            writer.insert(key, v.as_bytes()).unwrap();
+        });
+        let bytes = writer.into_inner().unwrap();
 
         self.documents_ids.insert(document_id);
-        self.documents_writer.insert(document_id.to_be_bytes(), record)?;
+        self.documents_writer.insert(document_id.to_be_bytes(), bytes)?;
         Self::write_docid_word_positions(&mut self.docid_word_positions_writer, document_id, words_positions)?;
 
         Ok(())

src/subcommand/search.rs

@@ -68,7 +68,7 @@ pub fn run(opt: Opt) -> anyhow::Result<()> {
         let mut wtr = csv::Writer::from_writer(io::stdout());
         wtr.write_record(&headers)?;
         for (_id, record) in documents {
-            wtr.write_record(&record)?;
+            wtr.write_record(record.iter().map(|(_, v)| v))?;
         }
         wtr.flush()?;
 

src/subcommand/serve.rs

@@ -1,5 +1,6 @@
 use std::collections::HashSet;
 use std::fs::{File, create_dir_all};
+use std::mem;
 use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::str::FromStr;
@@ -7,9 +8,10 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use askama_warp::Template;
-use futures::{FutureExt, StreamExt};
 use futures::stream;
+use futures::{FutureExt, StreamExt};
 use heed::EnvOpenOptions;
+use indexmap::IndexMap;
 use serde::{Serialize, Deserialize};
 use structopt::StructOpt;
 use tokio::fs::File as TFile;
@@ -56,25 +58,21 @@ pub struct Opt {
     indexer: IndexerOpt,
 }
 
-fn highlight_record(record: &csv::StringRecord, words: &HashSet<String>) -> csv::StringRecord {
+fn highlight_record(record: &mut IndexMap<String, String>, words: &HashSet<String>) {
-    let mut output_record = csv::StringRecord::new();
+    for (_key, value) in record.iter_mut() {
-    let mut buffer = String::new();
+        let old_value = mem::take(value);
-    for field in record {
+        for (token_type, token) in simple_tokenizer(&old_value) {
-        buffer.clear();
-        for (token_type, token) in simple_tokenizer(field) {
             if token_type == TokenType::Word {
                 let lowercase_token = token.to_lowercase();
                 let to_highlight = words.contains(&lowercase_token);
-                if to_highlight { buffer.push_str("<mark>") }
+                if to_highlight { value.push_str("<mark>") }
-                buffer.push_str(token);
+                value.push_str(token);
-                if to_highlight { buffer.push_str("</mark>") }
+                if to_highlight { value.push_str("</mark>") }
             } else {
-                buffer.push_str(token);
+                value.push_str(token);
             }
         }
-        output_record.push_field(&buffer);
     }
-    output_record
 }
 
 #[derive(Template)]
@@ -327,13 +325,6 @@ pub fn run(opt: Opt) -> anyhow::Result<()> {
             .body(include_str!("../../public/jquery-3.4.1.min.js"))
         );
 
-    let dash_papaparse_route = warp::filters::method::get()
-        .and(warp::path!("papaparse.min.js"))
-        .map(|| Response::builder()
-            .header("content-type", "application/javascript; charset=utf-8")
-            .body(include_str!("../../public/papaparse.min.js"))
-        );
-
     let dash_filesize_route = warp::filters::method::get()
         .and(warp::path!("filesize.min.js"))
         .map(|| Response::builder()
@@ -390,32 +381,29 @@ pub fn run(opt: Opt) -> anyhow::Result<()> {
 
             let SearchResult { found_words, documents_ids } = search.execute().unwrap();
 
-            let body = match index.headers(&rtxn).unwrap() {
+            let mut documents = Vec::new();
-                Some(headers) => {
+            if let Some(headers) = index.headers(&rtxn).unwrap() {
-                    let mut wtr = csv::Writer::from_writer(Vec::new());
+                for (_id, record) in index.documents(&rtxn, documents_ids).unwrap() {
+                    let mut record = record.iter()
+                        .map(|(key_id, value)| {
+                            let key = headers[key_id as usize].to_owned();
+                            let value = std::str::from_utf8(value).unwrap().to_owned();
+                            (key, value)
+                        })
+                        .collect();
 
-                    // We write the headers
+                    if !disable_highlighting {
-                    wtr.write_record(&headers).unwrap();
+                        highlight_record(&mut record, &found_words);
-
-                    let documents = index.documents(&rtxn, documents_ids).unwrap();
-                    for (_id, record) in documents {
-                        let record = if disable_highlighting {
-                            record
-                        } else {
-                            highlight_record(&record, &found_words)
-                        };
-                        wtr.write_record(&record).unwrap();
                     }
 
-                    wtr.into_inner().unwrap()
+                    documents.push(record);
-                },
+                }
-                None => Vec::new(),
+            }
-            };
 
             Response::builder()
-                .header("Content-Type", "text/csv")
+                .header("Content-Type", "application/json")
                 .header("Time-Ms", before_search.elapsed().as_millis().to_string())
-                .body(String::from_utf8(body).unwrap())
+                .body(serde_json::to_string(&documents).unwrap())
         });
 
     async fn buf_stream(
@@ -504,7 +492,6 @@ pub fn run(opt: Opt) -> anyhow::Result<()> {
         .or(dash_bulma_dark_route)
         .or(dash_style_route)
         .or(dash_jquery_route)
-        .or(dash_papaparse_route)
         .or(dash_filesize_route)
         .or(dash_script_route)
         .or(updates_script_route)

templates/index.html

@@ -7,7 +7,6 @@
   <link rel="stylesheet" href="/bulma-prefers-dark.min.css">
   <link rel="stylesheet" href="/style.css">
   <script type="text/javascript" src="/jquery-3.4.1.min.js"></script>
-  <script type="text/javascript" src="/papaparse.min.js"></script>
   <script type="text/javascript" src="/filesize.min.js"></script>
   <title>{{ db_name }} | The milli engine</title>
 </head>

templates/updates.html

@@ -7,7 +7,6 @@
   <link rel="stylesheet" href="/bulma-prefers-dark.min.css">
   <link rel="stylesheet" href="/style.css">
   <script type="text/javascript" src="/jquery-3.4.1.min.js"></script>
-  <script type="text/javascript" src="/papaparse.min.js"></script>
   <script type="text/javascript" src="/filesize.min.js"></script>
   <title>{{ db_name }} | Updates</title>
 </head>