review fixes

.github/workflows/create_artifacts.yml

@@ -28,7 +28,7 @@ jobs:
           rust-version: stable
       - uses: actions/checkout@v1
       - name: Build
-        run: cargo build --release --locked --features mini-dashboard
+        run: cargo build --release --locked
       - name: Upload binaries to release
         uses: svenstaro/upload-release-action@v1-release
         with:

Dockerfile

@@ -20,7 +20,7 @@ ENV     RUSTFLAGS="-C target-feature=-crt-static"
 # Create dummy main.rs files for each workspace member to be able to compile all the dependencies
 RUN     find . -type d -name "meilisearch-*" | xargs -I{} sh -c 'mkdir {}/src; echo "fn main() { }" > {}/src/main.rs;'
 # Use `cargo build` instead of `cargo vendor` because we need to not only download but compile dependencies too
-RUN     $HOME/.cargo/bin/cargo build --release --features mini-dashboard
+RUN     $HOME/.cargo/bin/cargo build --release
 # Cleanup dummy main.rs files
 RUN     find . -path "*/src/main.rs" -delete
 
@@ -29,7 +29,7 @@ ARG     COMMIT_DATE
 ENV     COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE}
 
 COPY    . .
-RUN     $HOME/.cargo/bin/cargo build --release --features mini-dashboard
+RUN     $HOME/.cargo/bin/cargo build --release
 
 # Run
 FROM    alpine:3.10

meilisearch-http/Cargo.toml

@@ -11,15 +11,15 @@ name = "meilisearch"
 path = "src/main.rs"
 
 [build-dependencies]
-actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e" }
-anyhow = "*"
-cargo_toml = "0.9.0"
-hex = "0.4.3"
-reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false}
-sha-1 = "0.9.4"
-tempfile = "3.1.0"
+actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true }
+anyhow = { version = "*", optional = true }
+cargo_toml = { version = "0.9.0", optional = true }
+hex = { version = "0.4.3", optional = true }
+reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
+sha-1 = { version = "0.9.4", optional = true }
+tempfile = { version = "3.1.0", optional = true }
 vergen = "3.1.0"
-zip = "0.5.12"
+zip = { version = "0.5.12", optional = true }
 
 [dependencies]
 actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "2dac1a4"}
@@ -71,7 +71,7 @@ tar = "0.4.29"
 tempfile = "3.1.0"
 thiserror = "1.0.24"
 tokio = { version = "1", features = ["full"] }
-uuid = "0.8.2"
+uuid = { version =  "0.8.2", features = ["serde"] }
 walkdir = "2.3.2"
 obkv = "0.1.1"
 
@@ -101,8 +101,18 @@ tempdir = "0.3.7"
 urlencoding = "1.1.1"
 
 [features]
-mini-dashboard = ["default", "actix-web-static-files"]
-default = ["sentry"]
+mini-dashboard = [
+    "actix-web-static-files",
+    "anyhow",
+    "cargo_toml",
+    "hex",
+    "reqwest",
+    "sha-1",
+    "tempfile",
+    "zip",
+]
+telemetry = ["sentry"]
+default = ["telemetry", "mini-dashboard"]
 
 [target.'cfg(target_os = "linux")'.dependencies]
 jemallocator = "0.3.2"

meilisearch-http/build.rs

@@ -1,15 +1,4 @@
-use std::env;
-use std::fs::create_dir_all;
-use std::io::Cursor;
-use std::path::PathBuf;
-
-use anyhow::Context;
-use sha1::{Sha1, Digest};
-use reqwest::blocking::get;
-use actix_web_static_files::resource_dir;
-
 use vergen::{generate_cargo_keys, ConstantsFlags};
-use cargo_toml::Manifest;
 
 fn main() {
     // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag
@@ -19,15 +8,30 @@ fn main() {
     // Generate the 'cargo:' key output
     generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!");
 
-    if let Ok(_) = env::var("CARGO_FEATURE_MINI_DASHBOARD") {
-        setup_mini_dashboard().expect("Could not load mini-dashboard assets");
-    }
+    #[cfg(feature = "mini-dashboard")]
+    mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets");
 }
 
-fn setup_mini_dashboard() -> anyhow::Result<()> {
+#[cfg(feature = "mini-dashboard")]
+mod mini_dashboard {
+    use std::env;
+    use std::fs::{create_dir_all, File, OpenOptions};
+    use std::io::{Cursor, Read, Write};
+    use std::path::PathBuf;
+
+    use actix_web_static_files::resource_dir;
+    use anyhow::Context;
+    use cargo_toml::Manifest;
+    use reqwest::blocking::get;
+    use sha1::{Digest, Sha1};
+
+    pub fn setup_mini_dashboard() -> anyhow::Result<()> {
         let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
         let cargo_toml = cargo_manifest_dir.join("Cargo.toml");
 
+        let sha1_path = cargo_manifest_dir.join(".mini-dashboard.sha1");
+        let dashboard_dir = cargo_manifest_dir.join("mini-dashboard");
+
         let manifest = Manifest::from_path(cargo_toml).unwrap();
 
         let meta = &manifest
@@ -36,25 +40,45 @@ fn setup_mini_dashboard() -> anyhow::Result<()> {
             .context("package not specified in Cargo.toml")?
             .metadata
             .as_ref()
-        .context("no metadata specified in Cargo.toml")?
-        ["mini-dashboard"];
+            .context("no metadata specified in Cargo.toml")?["mini-dashboard"];
+
+        // Check if there already is a dashboard built, and if it is up to date.
+        if sha1_path.exists() && dashboard_dir.exists() {
+            let mut sha1_file = File::open(&sha1_path)?;
+            let mut sha1 = String::new();
+            sha1_file.read_to_string(&mut sha1)?;
+            if sha1 == meta["sha1"].as_str().unwrap() {
+                // Nothing to do.
+                return Ok(())
+            }
+        }
 
         let url = meta["assets-url"].as_str().unwrap();
 
-    let dashboard_assets_bytes = get(url)?
-        .bytes()?;
+        let dashboard_assets_bytes = get(url)?.bytes()?;
 
         let mut hasher = Sha1::new();
         hasher.update(&dashboard_assets_bytes);
-    let sha1_dashboard = hex::encode(hasher.finalize());
+        let sha1 = hex::encode(hasher.finalize());
 
-    assert_eq!(meta["sha1"].as_str().unwrap(), sha1_dashboard);
+        assert_eq!(meta["sha1"].as_str().unwrap(), sha1, "Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml");
 
-    let dashboard_dir = cargo_manifest_dir.join("mini-dashboard");
         create_dir_all(&dashboard_dir)?;
         let cursor = Cursor::new(&dashboard_assets_bytes);
         let mut zip = zip::read::ZipArchive::new(cursor)?;
         zip.extract(&dashboard_dir)?;
         resource_dir(&dashboard_dir).build()?;
+
+        // Write the sha1 for the dashboard back to file.
+        let mut file = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .truncate(true)
+            .open(sha1_path)?;
+
+        file.write_all(sha1.as_bytes())?;
+        file.flush()?;
+
         Ok(())
+    }
 }

meilisearch-http/public/interface.html

@@ -1,333 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="bulma.min.css">
-    <title>MeiliSearch</title>
-    <style>
-      em {
-        color: hsl(204, 86%, 25%);
-        font-style: inherit;
-        background-color: hsl(204, 86%, 88%);
-      }
-
-      #results {
-        max-width: 900px;
-        margin: 20px auto 0 auto;
-        padding: 0;
-      }
-
-      .notification {
-        display: flex;
-        justify-content: center;
-      }
-
-      .level-left {
-        margin-right: 50px;
-      }
-
-      .document {
-        border-radius: 4px;
-        margin-bottom: 20px;
-        display: flex;
-      }
-
-      .document ol {
-        flex: 0 0 75%;
-        max-width: 75%;
-        padding: 0;
-        margin: 0;
-        list-style-type: none;
-      }
-
-      .document ol li {
-        list-style: none;
-      }
-
-
-      .document .image {
-        max-width: 50%;
-        margin: 0 auto;
-        box-sizing: border-box;
-      }
-
-      @media screen and (min-width: 770px) {
-        .document .image {
-          max-width: 25%;
-          flex: 0 0 25%;
-          margin: 0;
-          padding-left: 30px;
-          box-sizing: border-box;
-        }
-      }
-
-      .document .image img {
-        width: 100%;
-      }
-
-      .attribute {
-        text-align: center;
-        box-sizing: border-box;
-        text-transform: uppercase;
-        font-weight: bold;
-        color: rgba(0,0,0,.7);
-      }
-
-      @media screen and (min-width: 770px) {
-        .attribute {
-          flex: 0 0 25%;
-          max-width: 25%;
-          text-align: right;
-          padding-right: 10px;
-          font-weight: normal;
-          box-sizing: border-box;
-        }
-      }
-      @media screen and (max-width: 770px) {
-        .attribute {
-          padding-bottom: 0;
-        }
-      }
-
-      .content {
-        flex: 0 0 75%;
-        box-sizing: border-box;
-        color: rgba(0,0,0,.9);
-        overflow-wrap: anywhere;
-      }
-
-      .hero-foot {
-        padding-bottom: 3rem;
-      }
-
-      @media screen and (max-width: 770px) {
-        .align-on-mobile {
-          text-align: center;
-        }
-      }
-    </style>
-  </head>
-  <body>
-
-    <section class="hero is-light">
-      
-      <div class="hero-body">
-        <div class="container">
-          <div class="content is-medium align-on-mobile">
-            <h1 class="title is-1 is-spaced">
-              Welcome to MeiliSearch
-            </h1>
-            <p class="subtitle is-4">
-              This dashboard will help you check the search results with ease.
-            </p>
-          </div>
-          <div class="columns">
-            <div class="column is-4">
-              <div class="field">
-                  <!-- API Key -->
-                    <label class="label" for="apiKey">API Key (optionnal)</label>
-                    <div class="control">
-                      <input id="apiKey" class="input is-small" type="password" placeholder="Enter your API key">
-                    </div>
-                    <p class="help">At least a private API key is required for the dashboard to access the indexes list.</p>
-              </div>
-            </div>
-          </div>
-          <div class="columns">
-            <div class="column is-8">
-                <label class="label" for="search">Search something</label>
-                <div class="field has-addons">
-                  <div class="control">
-                    <span class="select">
-                      <select role="listbox" id="index" aria-label="Select the index you want to search on">
-                        <!-- indexes names -->
-                      </select>
-                    </span>
-                  </div>
-                  <div class="control is-expanded">
-                    <input id="search" class="input" type="search" autofocus placeholder="e.g. George Clooney" aria-label="Search through your documents">
-                  </div>
-                </div>
-            </div>
-            <div class="column is-4">
-              <div class="columns">
-                <div class="column is-6 has-text-centered">
-                  <p class="heading">Documents</p>
-                  <p id="count" class="title">0</p>
-                </div>
-                <div class="column is-6 has-text-centered">
-                  <p class="heading">Time Spent</p>
-                  <p id="time" class="title">N/A</p>
-                </div>
-              </div>
-            </div>
-          </div>
-        </div>
-      </div>
-    </section>
-
-    <section>
-      <div class="container">
-        <ol id="results" class="content">
-          <!-- documents matching resquests -->
-        </ol>
-      </div>
-    </section>
-  </body>
-
-  <script>
-    function sanitizeHTMLEntities(str) {
-        if (str && typeof str === 'string') {
-            str = str.replace(/</g,"&lt;");
-            str = str.replace(/>/g,"&gt;");
-            str = str.replace(/&lt;em&gt;/g,"<em>");
-            str = str.replace(/&lt;\/em&gt;/g,"<\/em>");
-        }
-        return str;
-    }
-
-    function httpGet(theUrl, apiKey) {
-        var xmlHttp = new XMLHttpRequest();
-        xmlHttp.open("GET", theUrl, false); // false for synchronous request
-        if (apiKey) {
-          xmlHttp.setRequestHeader("x-Meili-API-Key", apiKey);
-        }
-        xmlHttp.send(null);
-        return xmlHttp.responseText;
-    }
-
-    function refreshIndexList() {
-        // TODO we must not block here
-        let result = JSON.parse(httpGet(`${baseUrl}/indexes`, localStorage.getItem('apiKey')));
-
-        if (!Array.isArray(result)) { return }
-
-        let select = document.getElementById("index");
-        select.innerHTML = '';
-
-        for (index of result) {
-            const option = document.createElement('option');
-            option.value = index.uid;
-            option.innerHTML = index.name;
-            select.appendChild(option);
-        }
-    }
-
-    let lastRequest = undefined;
-
-    function triggerSearch() {
-        var e = document.getElementById("index");
-        if (e.selectedIndex == -1) { return }
-        var index = e.options[e.selectedIndex].value;
-
-        let theUrl = `${baseUrl}/indexes/${index}/search?q=${encodeURIComponent(search.value)}&attributesToHighlight=*`;
-
-        if (lastRequest) { lastRequest.abort() }
-        lastRequest = new XMLHttpRequest();
-
-        lastRequest.open("GET", theUrl, true);
-
-        if (localStorage.getItem('apiKey')) {
-          lastRequest.setRequestHeader("x-Meili-API-Key", localStorage.getItem('apiKey'));
-        }
-
-        lastRequest.onload = function (e) {
-            if (lastRequest.readyState === 4 && lastRequest.status === 200) {
-                let sanitizedResponseText = sanitizeHTMLEntities(lastRequest.responseText);
-                let httpResults = JSON.parse(sanitizedResponseText);
-                results.innerHTML = '';
-
-                let processingTimeMs = httpResults.processingTimeMs;
-                let numberOfDocuments = httpResults.nbHits;
-                time.innerHTML = `${processingTimeMs}ms`;
-                count.innerHTML = `${numberOfDocuments}`;
-
-                for (result of httpResults.hits) {
-                    const element = {...result, ...result._formatted };
-                    delete element._formatted;
-
-                    const elem = document.createElement('li');
-                    elem.classList.add("document","box");
-
-                    const div = document.createElement('div');
-                    div.classList.add("columns","is-desktop","is-tablet");
-                    const info = document.createElement('div');
-                    info.classList.add("column","align-on-mobile");
-                    let image = undefined;
-
-                    for (const prop in element) {
-                        // Check if property is an image url link.
-                        if (typeof result[prop] === 'string') {
-                            if (image == undefined && result[prop].match(/^(https|http):\/\/.*(jpe?g|png|gif)(\?.*)?$/g)) {
-                                image = result[prop];
-                            }
-                        }
-
-                        const field = document.createElement('div');
-                        field.classList.add("columns");
-
-                        const attribute = document.createElement('div');
-                        attribute.classList.add("attribute", "column");
-                        attribute.innerHTML = prop;
-
-                        const content = document.createElement('div');
-                        content.classList.add("content", "column");
-                        
-                        if (typeof (element[prop]) === "object") {
-                          content.innerHTML = JSON.stringify(element[prop]);
-                        } else {
-                          content.innerHTML = element[prop];
-                        }
-
-                        field.appendChild(attribute);
-                        field.appendChild(content);
-
-                        info.appendChild(field);
-                    }
-
-                    div.appendChild(info);
-                    elem.appendChild(div);
-
-                    if (image != undefined) {
-                        const divImage = document.createElement('div');
-                        divImage.classList.add("image","column","align-on-mobile");
-                        
-                        const img = document.createElement('img');
-                        img.src = image;
-                        img.setAttribute("alt","Item illustration");
-
-                        divImage.appendChild(img);
-                        div.appendChild(divImage);
-                        elem.appendChild(div);
-                    }
-
-                    results.appendChild(elem)
-                }
-            } else {
-                console.error(lastRequest.statusText);
-            }
-        };
-        lastRequest.send(null);
-    }
-
-    if (!apiKey.value) {
-        apiKey.value = localStorage.getItem('apiKey');
-    }
-
-    apiKey.addEventListener('input', function(e) {
-        localStorage.setItem('apiKey', apiKey.value);
-        refreshIndexList();
-    }, false);
-
-    let baseUrl = window.location.origin;
-    refreshIndexList();
-
-    search.oninput = triggerSearch;
-    
-    let select = document.getElementById("index");
-    select.onchange = triggerSearch;
-
-    triggerSearch();
-  </script>
-</html>

meilisearch-http/src/lib.rs

@@ -9,7 +9,6 @@ pub mod routes;
 pub use self::data::Data;
 pub use option::Opt;
 
-
 #[macro_export]
 macro_rules! create_app {
     ($data:expr, $enable_frontend:expr) => {{
@@ -52,7 +51,6 @@ macro_rules! create_app {
         #[cfg(feature = "mini-dashboard")]
         let app = if $enable_frontend {
             let generated = dashboard::generate();
-            let keys = generated.keys().collect::<Vec<_>>();
             let service = ResourceFiles::new("/", generated);
             app.service(service)
         } else {

meilisearch-http/src/main.rs

@@ -72,20 +72,14 @@ async fn main() -> Result<(), MainError> {
 
     print_launch_resume(&opt, &data);
 
-    let enable_frontend = opt.env != "production";
-
-    run_http(data, opt, enable_frontend).await?;
+    run_http(data, opt).await?;
 
     Ok(())
 }
 
-#[allow(unused_variables)]
-async fn run_http(
-    data: Data,
-    opt: Opt,
-    enable_frontend: bool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let http_server = HttpServer::new(move || create_app!(&data, enable_frontend))
+async fn run_http(data: Data, opt: Opt) -> Result<(), Box<dyn std::error::Error>> {
+    let _enable_dashboard = &opt.env == "development";
+    let http_server = HttpServer::new(move || create_app!(&data, _enable_dashboard))
         // Disable signals allows the server to terminate immediately when a user enter CTRL-C
         .disable_signals();
 
@@ -103,11 +97,11 @@ async fn run_http(
 pub fn print_launch_resume(opt: &Opt, data: &Data) {
     let commit_sha = match option_env!("COMMIT_SHA") {
         Some("") | None => env!("VERGEN_SHA"),
-        Some(commit_sha) => commit_sha
+        Some(commit_sha) => commit_sha,
     };
     let commit_date = match option_env!("COMMIT_DATE") {
         Some("") | None => env!("VERGEN_COMMIT_DATE"),
-        Some(commit_date) => commit_date
+        Some(commit_date) => commit_date,
     };
 
     let ascii_name = r#"

meilisearch-http/src/routes/mod.rs

@@ -28,14 +28,6 @@ impl IndexUpdateResponse {
     }
 }
 
-/// Return the dashboard, should not be used in production. See [running]
-#[get("/")]
-pub async fn load_html() -> HttpResponse {
-    HttpResponse::Ok()
-        .content_type("text/html; charset=utf-8")
-        .body(include_str!("../../public/interface.html").to_string())
-}
-
 /// Always return a 200 with:
 /// ```json
 /// {
@@ -46,10 +38,3 @@ pub async fn load_html() -> HttpResponse {
 pub async fn running() -> HttpResponse {
     HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" }))
 }
-
-#[get("/bulma.min.css")]
-pub async fn load_css() -> HttpResponse {
-    HttpResponse::Ok()
-        .content_type("text/css; charset=utf-8")
-        .body(include_str!("../../public/bulma.min.css").to_string())
-}