feat: Introduce a little http query server

This commit is contained in:
Clément Renault 2018-09-14 19:09:20 +02:00
parent 9ee71848bb
commit 3f503446d5
4 changed files with 1036 additions and 0 deletions

924
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -3,6 +3,7 @@ members = [
"raptor", "raptor",
"raptor-indexer", "raptor-indexer",
"raptor-search", "raptor-search",
"raptor-http",
] ]
[profile.release] [profile.release]

21
raptor-http/Cargo.toml Normal file
View File

@ -0,0 +1,21 @@
cargo-features = ["edition"]
[package]
edition = "2018"
name = "raptor-http"
version = "0.1.0"
authors = ["Clément Renault <renault.cle@gmail.com>"]
[dependencies]
raptor = { path = "../raptor" }
serde_derive = "1.0"
serde_json = "1.0"
serde = "1.0"
warp = "0.1"
[dependencies.fst]
git = "https://github.com/Kerollmops/fst.git"
branch = "automaton-for-deref"
[dependencies.rocksdb]
git = "https://github.com/pingcap/rust-rocksdb.git"

90
raptor-http/src/main.rs Normal file
View File

@ -0,0 +1,90 @@
#[macro_use] extern crate serde_derive;
use std::env;
use std::io::Write;
use std::sync::Arc;
use std::error::Error;
use std::str::from_utf8_unchecked;
use rocksdb::{DB, DBOptions, IngestExternalFileOptions};
use raptor::{automaton, Metadata, RankedStream};
use fst::Streamer;
use warp::Filter;
#[derive(Debug, Deserialize)]
struct SearchQuery { query: String }
#[derive(Debug, Serialize)]
struct Document<'a> {
id: u64,
title: &'a str,
description: &'a str,
}
fn search<M, D>(metadata: M, database: D, query: &str) -> Result<String, Box<Error>>
where M: AsRef<Metadata>,
D: AsRef<DB>,
{
let mut automatons = Vec::new();
for query in query.split_whitespace() {
let lev = automaton::build(query);
automatons.push(lev);
}
let mut stream = RankedStream::new(metadata.as_ref(), automatons, 20);
let mut body = Vec::new();
write!(&mut body, "[")?;
let mut first = true;
while let Some(document) = stream.next() {
let title_key = format!("{}-title", document.document_id);
let title = database.as_ref().get(title_key.as_bytes()).unwrap().unwrap();
let title = unsafe { from_utf8_unchecked(&title) };
let description_key = format!("{}-description", document.document_id);
let description = database.as_ref().get(description_key.as_bytes()).unwrap().unwrap();
let description = unsafe { from_utf8_unchecked(&description) };
let document = Document {
id: document.document_id,
title: title,
description: description,
};
if !first { write!(&mut body, ",")? }
serde_json::to_writer(&mut body, &document)?;
first = false;
}
write!(&mut body, "]")?;
Ok(String::from_utf8(body)?)
}
fn main() {
let name = env::args().nth(1).expect("Missing meta file name (e.g. lucid-ptolemy)");
let map_file = format!("{}.map", name);
let idx_file = format!("{}.idx", name);
let sst_file = format!("{}.sst", name);
let rocksdb = "rocksdb/storage";
let meta = unsafe { Metadata::from_paths(map_file, idx_file).unwrap() };
let meta = Arc::new(meta);
let db = DB::open_default(rocksdb).unwrap();
db.ingest_external_file(&IngestExternalFileOptions::new(), &[&sst_file]).unwrap();
drop(db);
let db = DB::open_for_read_only(DBOptions::default(), rocksdb, false).unwrap();
let db = Arc::new(db);
let routes = warp::path("search")
.and(warp::query())
.map(move |query: SearchQuery| {
let body = search(meta.clone(), db.clone(), &query.query).unwrap();
body
})
.with(warp::reply::with::header("Content-Type", "application/json"));
warp::serve(routes).run(([127, 0, 0, 1], 3030));
}