Move the http server into its own sub-module

.gitignore

@@ -1,4 +1,11 @@
+# the milli project is a library
 /target
+/Cargo.lock
+
+# the sub target folder
+http-ui/target
+
+# datasets
 *.csv
 *.mmdb
 *.svg

Cargo.toml

@@ -27,6 +27,7 @@ once_cell = "1.4.0"
 rayon = "1.3.1"
 ringtail = "0.3.0"
 roaring = "0.6.1"
+serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0.59", features = ["preserve_order"] }
 slice-group-by = "0.2.6"
 smallstr =  { version = "0.2.0", features = ["serde"] }
@@ -42,15 +43,6 @@ itertools = "0.9.0"
 log = "0.4.11"
 stderrlog = "0.5.0"
 
-# http server
-askama = "0.10.1"
-askama_warp = "0.10.0"
-bytes = "0.5.6"
-futures = "0.3.6"
-serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "0.2", features = ["full"] }
-warp = "0.2.2"
-
 [dev-dependencies]
 criterion = "0.3.3"
 

README.md

@@ -15,6 +15,7 @@ It currently only supports the proximity criterion.
 You can specify the number of threads to use to index documents and many other settings too.
 
 ```bash
+cd http-ui
 cargo run --release -- serve --db my-database.mdb -vvv --indexing-jobs 8
 ```
 

http-ui/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "http-ui"
+description = "The HTTP user interface of the milli search engine"
+version = "0.1.0"
+authors = ["Clément Renault <clement@meilisearch.com>"]
+edition = "2018"
+
+[dependencies]
+anyhow = "1.0.28"
+grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3eb7ad9" }
+heed = "0.10.1"
+indexmap = "1.6.0"
+memmap = "0.7.0"
+milli = { path = ".." }
+once_cell = "1.4.1"
+rayon = "1.5.0"
+structopt = { version = "0.3.14", default-features = false, features = ["wrap_help"] }
+tempfile = "3.1.0"
+
+# http server
+askama = "0.10.1"
+askama_warp = "0.10.0"
+bytes = "0.5.6"
+flate2 = "1.0.19"
+futures = "0.3.6"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0.59", features = ["preserve_order"] }
+tokio = { version = "0.2", features = ["full"] }
+warp = "0.2.2"
+
+# logging
+log = "0.4.11"
+stderrlog = "0.5.0"

http-ui/src/main.rs

@@ -25,9 +25,9 @@ use tokio::sync::broadcast;
 use warp::filters::ws::Message;
 use warp::{Filter, http::Response};
 
-use crate::tokenizer::{simple_tokenizer, TokenType};
+use milli::tokenizer::{simple_tokenizer, TokenType};
-use crate::update::{UpdateBuilder, IndexDocumentsMethod, UpdateFormat};
+use milli::update::{UpdateBuilder, IndexDocumentsMethod, UpdateFormat};
-use crate::{Index, UpdateStore, SearchResult};
+use milli::{Index, UpdateStore, SearchResult};
 
 static GLOBAL_THREAD_POOL: OnceCell<ThreadPool> = OnceCell::new();
 
@@ -201,7 +201,10 @@ where T: Deserialize<'de>,
     Deserialize::deserialize(deserializer).map(Some)
 }
 
-pub fn run(opt: Opt) -> anyhow::Result<()> {
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let opt = Opt::from_args();
+
     stderrlog::new()
         .verbosity(opt.verbose)
         .show_level(false)
@@ -429,63 +432,63 @@ pub fn run(opt: Opt) -> anyhow::Result<()> {
         .and(warp::path!("bulma.min.css"))
         .map(|| Response::builder()
             .header("content-type", "text/css; charset=utf-8")
-            .body(include_str!("../../public/bulma.min.css"))
+            .body(include_str!("../public/bulma.min.css"))
         );
 
     let dash_bulma_dark_route = warp::filters::method::get()
         .and(warp::path!("bulma-prefers-dark.min.css"))
         .map(|| Response::builder()
             .header("content-type", "text/css; charset=utf-8")
-            .body(include_str!("../../public/bulma-prefers-dark.min.css"))
+            .body(include_str!("../public/bulma-prefers-dark.min.css"))
         );
 
     let dash_style_route = warp::filters::method::get()
         .and(warp::path!("style.css"))
         .map(|| Response::builder()
             .header("content-type", "text/css; charset=utf-8")
-            .body(include_str!("../../public/style.css"))
+            .body(include_str!("../public/style.css"))
         );
 
     let dash_jquery_route = warp::filters::method::get()
         .and(warp::path!("jquery-3.4.1.min.js"))
         .map(|| Response::builder()
             .header("content-type", "application/javascript; charset=utf-8")
-            .body(include_str!("../../public/jquery-3.4.1.min.js"))
+            .body(include_str!("../public/jquery-3.4.1.min.js"))
         );
 
     let dash_filesize_route = warp::filters::method::get()
         .and(warp::path!("filesize.min.js"))
         .map(|| Response::builder()
             .header("content-type", "application/javascript; charset=utf-8")
-            .body(include_str!("../../public/filesize.min.js"))
+            .body(include_str!("../public/filesize.min.js"))
         );
 
     let dash_script_route = warp::filters::method::get()
         .and(warp::path!("script.js"))
         .map(|| Response::builder()
             .header("content-type", "application/javascript; charset=utf-8")
-            .body(include_str!("../../public/script.js"))
+            .body(include_str!("../public/script.js"))
         );
 
     let updates_script_route = warp::filters::method::get()
         .and(warp::path!("updates-script.js"))
         .map(|| Response::builder()
             .header("content-type", "application/javascript; charset=utf-8")
-            .body(include_str!("../../public/updates-script.js"))
+            .body(include_str!("../public/updates-script.js"))
         );
 
     let dash_logo_white_route = warp::filters::method::get()
         .and(warp::path!("logo-white.svg"))
         .map(|| Response::builder()
             .header("content-type", "image/svg+xml")
-            .body(include_str!("../../public/logo-white.svg"))
+            .body(include_str!("../public/logo-white.svg"))
         );
 
     let dash_logo_black_route = warp::filters::method::get()
         .and(warp::path!("logo-black.svg"))
         .map(|| Response::builder()
             .header("content-type", "image/svg+xml")
-            .body(include_str!("../../public/logo-black.svg"))
+            .body(include_str!("../public/logo-black.svg"))
         );
 
     #[derive(Deserialize)]
@@ -568,6 +571,7 @@ pub fn run(opt: Opt) -> anyhow::Result<()> {
             UpdateFormat::Csv => String::from("csv"),
             UpdateFormat::Json => String::from("json"),
             UpdateFormat::JsonStream => String::from("json-stream"),
+            _ => panic!("Unknown update format"),
         };
 
         let meta = UpdateMeta::DocumentsAddition { method, format };
@@ -709,13 +713,5 @@ pub fn run(opt: Opt) -> anyhow::Result<()> {
         .or(update_ws_route);
 
     let addr = SocketAddr::from_str(&opt.http_listen_addr)?;
-    tokio::runtime::Builder::new()
+    Ok(warp::serve(routes).run(addr).await)
-        .threaded_scheduler()
-        .enable_all()
-        .build()?
-        .block_on(async {
-            warp::serve(routes).run(addr).await
-        });
-
-    Ok(())
 }

public/logo-black.svg

@@ -1,6 +0,0 @@
-<svg width="277" height="236" viewBox="0 0 277 236" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M213.085 190L242.907 86H276.196L246.375 190H213.085Z" fill="#494949"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M0 190L29.8215 86H63.1111L33.2896 190H0Z" fill="#494949"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M124.986 0L57.5772 235.083L60.7752 236H90.6038L158.276 0H124.986Z" fill="#494949"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M195.273 0L127.601 236H160.891L228.563 0H195.273Z" fill="#494949"/>
-</svg>

public/logo-white.svg

@@ -1,6 +0,0 @@
-<svg width="277" height="236" viewBox="0 0 277 236" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M213.508 190L243.33 86H276.619L246.798 190H213.508Z" fill="#B5B5B5"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M0.422791 190L30.2443 86H63.5339L33.7124 190H0.422791Z" fill="#B5B5B5"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M125.409 0L58 235.083L61.198 236H91.0266L158.699 0H125.409Z" fill="#B5B5B5"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M195.696 0L128.024 236H161.313L228.985 0H195.696Z" fill="#B5B5B5"/>
-</svg>

src/main.rs

@@ -1,7 +1,6 @@
 use structopt::StructOpt;
 
 use milli::subcommand::infos::{self, Opt as InfosOpt};
-use milli::subcommand::serve::{self, Opt as ServeOpt};
 use milli::subcommand::search::{self, Opt as SearchOpt};
 
 #[cfg(target_os = "linux")]
@@ -11,14 +10,12 @@ static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 #[derive(Debug, StructOpt)]
 #[structopt(name = "milli", about = "The milli project.")]
 enum Command {
-    Serve(ServeOpt),
     Infos(InfosOpt),
     Search(SearchOpt),
 }
 
 fn main() -> anyhow::Result<()> {
     match Command::from_args() {
-        Command::Serve(opt) => serve::run(opt),
         Command::Infos(opt) => infos::run(opt),
         Command::Search(opt) => search::run(opt),
     }

src/subcommand/mod.rs

@@ -1,3 +1,2 @@
 pub mod infos;
 pub mod search;
-pub mod serve;