Merge remote-tracking branch 'transplant/main'

.dockerignore

@@ -0,0 +1,5 @@
+target
+Dockerfile
+.dockerignore
+.git
+.gitignore

.github/release-draft-template.yml

@@ -0,0 +1,13 @@
+name-template: 'v$RESOLVED_VERSION'
+tag-template: 'v$RESOLVED_VERSION'
+version-template: '0.21.0-alpha.$PATCH'
+exclude-labels:
+  - 'skip-changelog'
+template: |
+  ## Changes
+
+  $CHANGES
+no-changes-template: 'Changes are coming soon 😎'
+sort-direction: 'ascending'
+version-resolver:
+  default: patch

.github/workflows/create_artifacts.yml

@@ -0,0 +1,38 @@
+name: Create artifacts
+
+on:
+  push:
+    tags:
+      - v*-alpha.*
+
+jobs:
+  nightly:
+    name: Build Nighlty ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+          - os: ubuntu-latest
+            artifact_name: meilisearch
+            asset_name: meilisearch-alpha-linux-amd64
+          - os: macos-latest
+            artifact_name: meilisearch
+            asset_name: meilisearch-alpha-macos-amd64
+          - os: windows-latest
+            artifact_name: meilisearch.exe
+            asset_name: meilisearch-alpha-windows-amd64.exe
+    steps:
+      - uses: hecrj/setup-rust-action@master
+        with:
+          rust-version: stable
+      - uses: actions/checkout@v1
+      - name: Build
+        run: cargo build --release --locked
+      - name: Upload binaries to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: target/release/${{ matrix.artifact_name }}
+          asset_name: ${{ matrix.asset_name }}
+          tag: ${{ github.ref }}

.github/workflows/flaky.yml

@@ -0,0 +1,15 @@
+name: Look for flaky tests
+on:
+  schedule:
+    - cron: "0 12 * * FRI" # every friday at 12:00PM
+
+jobs:
+  flaky:
+    runs-on: ubuntu-18.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install cargo-flaky
+      run: cargo install cargo-flaky
+    - name: Run cargo flaky 100 times
+      run: cargo flaky -i 100 --release

.github/workflows/publish_to_docker.yml

@@ -0,0 +1,26 @@
+name: Publish to dockerhub
+
+on:
+  push:
+    tags:
+      - v*-alpha.*
+
+jobs:
+  publish:
+    name: Publishing to dockerhub
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set COMMIT_DATE env variable
+        run: |
+          echo "COMMIT_DATE=$( git log --pretty=format:'%ad' -n1 --date=short )" >> $GITHUB_ENV
+      - name: Publish to Registry
+        uses: elgohr/Publish-Docker-Github-Action@master
+        env:
+          COMMIT_SHA: ${{ github.sha }}
+        with:
+          name: getmeili/meilisearch
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          tag_names: true
+          buildargs: COMMIT_SHA,COMMIT_DATE

.github/workflows/release-drafter.yml

@@ -0,0 +1,16 @@
+name: Release Drafter
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  update_release_draft:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: release-drafter/release-drafter@v5
+        with:
+          config-name: release-draft-template.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }}

.github/workflows/rust.yml

@@ -0,0 +1,81 @@
+name: Rust
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    # trying and staging branches are for Bors config
+    branches:
+      - trying
+      - staging
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  tests:
+    name: Tests on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-latest]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release
+
+  # We don't run test on Windows since we get the following error: There is not enough space on the disk.
+  check-on-windows:
+    name: Cargo check on Windows
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: check
+        args: --no-default-features
+    - name: Run cargo check with all default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: check
+
+  clippy:
+    name: Run Clippy
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: clippy
+      - name: Run cargo clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-targets -- --deny warnings
+
+  fmt:
+    name: Run Rustfmt
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+      - name: Run cargo fmt
+        run: cargo fmt --all -- --check

.gitignore

@@ -0,0 +1,7 @@
+/target
+**/*.csv
+**/*.json_lines
+**/*.rs.bk
+/*.mdb
+/query-history.txt
+/data.ms

Cargo.toml

@@ -0,0 +1,8 @@
+[workspace]
+members = [
+    "meilisearch-http",
+    "meilisearch-error",
+]
+
+[profile.release]
+debug = true

Dockerfile

@@ -0,0 +1,45 @@
+# Compile
+FROM    alpine:3.14 AS compiler
+
+RUN     apk update --quiet
+RUN     apk add curl
+RUN     apk add build-base
+
+RUN     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+WORKDIR /meilisearch
+
+COPY    Cargo.lock .
+COPY    Cargo.toml .
+
+COPY    meilisearch-error/Cargo.toml meilisearch-error/
+COPY    meilisearch-http/Cargo.toml meilisearch-http/
+
+ENV     RUSTFLAGS="-C target-feature=-crt-static"
+
+# Create dummy main.rs files for each workspace member to be able to compile all the dependencies
+RUN     find . -type d -name "meilisearch-*" | xargs -I{} sh -c 'mkdir {}/src; echo "fn main() { }" > {}/src/main.rs;'
+# Use `cargo build` instead of `cargo vendor` because we need to not only download but compile dependencies too
+RUN     $HOME/.cargo/bin/cargo build --release
+# Cleanup dummy main.rs files
+RUN     find . -path "*/src/main.rs" -delete
+
+ARG     COMMIT_SHA
+ARG     COMMIT_DATE
+ENV     COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE}
+
+COPY    . .
+RUN     $HOME/.cargo/bin/cargo build --release
+
+# Run
+FROM    alpine:3.14
+
+RUN     apk add -q --no-cache libgcc tini
+
+COPY    --from=compiler /meilisearch/target/release/meilisearch .
+
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
+CMD     ./meilisearch

bors.toml

@@ -0,0 +1,9 @@
+status = [
+    'Tests on ubuntu-18.04',
+    'Tests on macos-latest',
+    'Cargo check on Windows',
+    'Run Clippy',
+    'Run Rustfmt'
+]
+# 3 hours timeout
+timeout-sec = 10800

meilisearch-error/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "meilisearch-error"
+version = "0.19.0"
+authors = ["marin <postma.marin@protonmail.com>"]
+edition = "2018"
+
+[dependencies]
+actix-http = "=3.0.0-beta.6"

meilisearch-error/src/lib.rs

@@ -0,0 +1,200 @@
+use std::fmt;
+
+use actix_http::http::StatusCode;
+
+pub trait ErrorCode: std::error::Error {
+    fn error_code(&self) -> Code;
+
+    /// returns the HTTP status code ascociated with the error
+    fn http_status(&self) -> StatusCode {
+        self.error_code().http()
+    }
+
+    /// returns the doc url ascociated with the error
+    fn error_url(&self) -> String {
+        self.error_code().url()
+    }
+
+    /// returns error name, used as error code
+    fn error_name(&self) -> String {
+        self.error_code().name()
+    }
+
+    /// return the error type
+    fn error_type(&self) -> String {
+        self.error_code().type_()
+    }
+}
+
+#[allow(clippy::enum_variant_names)]
+enum ErrorType {
+    InternalError,
+    InvalidRequestError,
+    AuthenticationError,
+}
+
+impl fmt::Display for ErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use ErrorType::*;
+
+        match self {
+            InternalError => write!(f, "internal_error"),
+            InvalidRequestError => write!(f, "invalid_request_error"),
+            AuthenticationError => write!(f, "authentication_error"),
+        }
+    }
+}
+
+pub enum Code {
+    // index related error
+    CreateIndex,
+    IndexAlreadyExists,
+    IndexNotFound,
+    InvalidIndexUid,
+    OpenIndex,
+
+    // invalid state error
+    InvalidState,
+    MissingPrimaryKey,
+    PrimaryKeyAlreadyPresent,
+
+    MaxFieldsLimitExceeded,
+    MissingDocumentId,
+
+    Facet,
+    Filter,
+
+    BadParameter,
+    BadRequest,
+    DocumentNotFound,
+    Internal,
+    InvalidToken,
+    MissingAuthorizationHeader,
+    NotFound,
+    PayloadTooLarge,
+    RetrieveDocument,
+    SearchDocuments,
+    UnsupportedMediaType,
+
+    DumpAlreadyInProgress,
+    DumpProcessFailed,
+}
+
+impl Code {
+    /// ascociate a `Code` variant to the actual ErrCode
+    fn err_code(&self) -> ErrCode {
+        use Code::*;
+
+        match self {
+            // index related errors
+            // create index is thrown on internal error while creating an index.
+            CreateIndex => ErrCode::internal("index_creation_failed", StatusCode::BAD_REQUEST),
+            IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::BAD_REQUEST),
+            // thrown when requesting an unexisting index
+            IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND),
+            InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST),
+            OpenIndex => {
+                ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR)
+            }
+
+            // invalid state error
+            InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR),
+            // thrown when no primary key has been set
+            MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST),
+            // error thrown when trying to set an already existing primary key
+            PrimaryKeyAlreadyPresent => {
+                ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST)
+            }
+
+            // invalid document
+            MaxFieldsLimitExceeded => {
+                ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST)
+            }
+            MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST),
+
+            // error related to facets
+            Facet => ErrCode::invalid("invalid_facet", StatusCode::BAD_REQUEST),
+            // error related to filters
+            Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST),
+
+            BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
+            BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
+            DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
+            Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
+            InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN),
+            MissingAuthorizationHeader => {
+                ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED)
+            }
+            NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND),
+            PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE),
+            RetrieveDocument => {
+                ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST)
+            }
+            SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST),
+            UnsupportedMediaType => {
+                ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
+            }
+
+            // error related to dump
+            DumpAlreadyInProgress => {
+                ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT)
+            }
+            DumpProcessFailed => {
+                ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR)
+            }
+        }
+    }
+
+    /// return the HTTP status code ascociated with the `Code`
+    fn http(&self) -> StatusCode {
+        self.err_code().status_code
+    }
+
+    /// return error name, used as error code
+    fn name(&self) -> String {
+        self.err_code().error_name.to_string()
+    }
+
+    /// return the error type
+    fn type_(&self) -> String {
+        self.err_code().error_type.to_string()
+    }
+
+    /// return the doc url ascociated with the error
+    fn url(&self) -> String {
+        format!("https://docs.meilisearch.com/errors#{}", self.name())
+    }
+}
+
+/// Internal structure providing a convenient way to create error codes
+struct ErrCode {
+    status_code: StatusCode,
+    error_type: ErrorType,
+    error_name: &'static str,
+}
+
+impl ErrCode {
+    fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::AuthenticationError,
+        }
+    }
+
+    fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InternalError,
+        }
+    }
+
+    fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InvalidRequestError,
+        }
+    }
+}

meilisearch-http/Cargo.toml

@@ -0,0 +1,123 @@
+[package]
+authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
+description = "MeiliSearch HTTP server"
+edition = "2018"
+license = "MIT"
+name = "meilisearch-http"
+version = "0.21.0-alpha.6"
+
+[[bin]]
+name = "meilisearch"
+path = "src/main.rs"
+
+[build-dependencies]
+actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true }
+anyhow = { version = "*", optional = true }
+cargo_toml = { version = "0.9.0", optional = true }
+hex = { version = "0.4.3", optional = true }
+reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
+sha-1 = { version = "0.9.4", optional = true }
+tempfile = { version = "3.1.0", optional = true }
+vergen = "3.1.0"
+zip = { version = "0.5.12", optional = true }
+
+[dependencies]
+actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "2dac1a4"}
+actix-http = { version = "=3.0.0-beta.6" }
+actix-service = "2.0.0"
+actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] }
+actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true }
+anyhow = "1.0.36"
+async-stream = "0.3.0"
+async-trait = "0.1.42"
+arc-swap = "1.2.0"
+byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
+bytes = "0.6.0"
+chrono = { version = "0.4.19", features = ["serde"] }
+crossbeam-channel = "0.5.0"
+either = "1.6.1"
+env_logger = "0.8.2"
+flate2 = "1.0.19"
+fst = "0.4.5"
+futures = "0.3.7"
+futures-util = "0.3.8"
+grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
+http = "0.2.1"
+indexmap = { version = "1.3.2", features = ["serde-1"] }
+itertools = "0.10.0"
+log = "0.4.8"
+main_error = "0.1.0"
+meilisearch-error = { path = "../meilisearch-error" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" }
+memmap = "0.7.0"
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.0" }
+mime = "0.3.16"
+num_cpus = "1.13.0"
+once_cell = "1.5.2"
+oxidized-json-checker = "0.3.2"
+parking_lot = "0.11.1"
+rand = "0.7.3"
+rayon = "1.5.0"
+regex = "1.4.2"
+rustls = "0.19"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0.59", features = ["preserve_order"] }
+sha2 = "0.9.1"
+siphasher = "0.3.2"
+slice-group-by = "0.2.6"
+structopt = "0.3.20"
+tar = "0.4.29"
+tempfile = "3.1.0"
+thiserror = "1.0.24"
+tokio = { version = "1", features = ["full"] }
+uuid = { version =  "0.8.2", features = ["serde"] }
+walkdir = "2.3.2"
+obkv = "0.1.1"
+pin-project = "1.0.7"
+whoami = { version = "1.1.2", optional = true }
+reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true }
+
+[dependencies.sentry]
+default-features = false
+features = [
+    "backtrace",
+    "contexts",
+    "panic",
+    "reqwest",
+    "rustls",
+    "log",
+]
+optional = true
+version = "0.22.0"
+
+
+[dev-dependencies]
+actix-rt = "2.1.0"
+assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
+mockall = "0.9.1"
+paste = "1.0.5"
+serde_url_params = "0.2.0"
+tempdir = "0.3.7"
+urlencoding = "1.1.1"
+
+[features]
+mini-dashboard = [
+    "actix-web-static-files",
+    "anyhow",
+    "cargo_toml",
+    "hex",
+    "reqwest",
+    "sha-1",
+    "tempfile",
+    "zip",
+]
+analytics = ["sentry", "whoami", "reqwest"]
+default = ["analytics", "mini-dashboard"]
+
+[target.'cfg(target_os = "linux")'.dependencies]
+jemallocator = "0.3.2"
+
+[package.metadata.mini-dashboard]
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.3/build.zip"
+sha1 = "fea1780e13d8e570e35a1921e7a45cabcd501d5e"

meilisearch-http/build.rs

@@ -0,0 +1,89 @@
+use vergen::{generate_cargo_keys, ConstantsFlags};
+
+fn main() {
+    // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag
+    let mut flags = ConstantsFlags::all();
+    flags.toggle(ConstantsFlags::SEMVER_FROM_CARGO_PKG);
+
+    // Generate the 'cargo:' key output
+    generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!");
+
+    #[cfg(feature = "mini-dashboard")]
+    mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets");
+}
+
+#[cfg(feature = "mini-dashboard")]
+mod mini_dashboard {
+    use std::env;
+    use std::fs::{create_dir_all, File, OpenOptions};
+    use std::io::{Cursor, Read, Write};
+    use std::path::PathBuf;
+
+    use actix_web_static_files::resource_dir;
+    use anyhow::Context;
+    use cargo_toml::Manifest;
+    use reqwest::blocking::get;
+    use sha1::{Digest, Sha1};
+
+    pub fn setup_mini_dashboard() -> anyhow::Result<()> {
+        let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+        let cargo_toml = cargo_manifest_dir.join("Cargo.toml");
+        let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+
+        let sha1_path = out_dir.join(".mini-dashboard.sha1");
+        let dashboard_dir = out_dir.join("mini-dashboard");
+
+        let manifest = Manifest::from_path(cargo_toml).unwrap();
+
+        let meta = &manifest
+            .package
+            .as_ref()
+            .context("package not specified in Cargo.toml")?
+            .metadata
+            .as_ref()
+            .context("no metadata specified in Cargo.toml")?["mini-dashboard"];
+
+        // Check if there already is a dashboard built, and if it is up to date.
+        if sha1_path.exists() && dashboard_dir.exists() {
+            let mut sha1_file = File::open(&sha1_path)?;
+            let mut sha1 = String::new();
+            sha1_file.read_to_string(&mut sha1)?;
+            if sha1 == meta["sha1"].as_str().unwrap() {
+                // Nothing to do.
+                return Ok(());
+            }
+        }
+
+        let url = meta["assets-url"].as_str().unwrap();
+
+        let dashboard_assets_bytes = get(url)?.bytes()?;
+
+        let mut hasher = Sha1::new();
+        hasher.update(&dashboard_assets_bytes);
+        let sha1 = hex::encode(hasher.finalize());
+
+        assert_eq!(
+            meta["sha1"].as_str().unwrap(),
+            sha1,
+            "Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml"
+        );
+
+        create_dir_all(&dashboard_dir)?;
+        let cursor = Cursor::new(&dashboard_assets_bytes);
+        let mut zip = zip::read::ZipArchive::new(cursor)?;
+        zip.extract(&dashboard_dir)?;
+        resource_dir(&dashboard_dir).build()?;
+
+        // Write the sha1 for the dashboard back to file.
+        let mut file = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .truncate(true)
+            .open(sha1_path)?;
+
+        file.write_all(sha1.as_bytes())?;
+        file.flush()?;
+
+        Ok(())
+    }
+}

meilisearch-http/src/analytics.rs

@@ -0,0 +1,126 @@
+use std::hash::{Hash, Hasher};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use log::debug;
+use serde::Serialize;
+use siphasher::sip::SipHasher;
+
+use crate::Data;
+use crate::Opt;
+
+const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47";
+
+#[derive(Debug, Serialize)]
+struct EventProperties {
+    database_size: u64,
+    last_update_timestamp: Option<i64>, //timestamp
+    number_of_documents: Vec<u64>,
+}
+
+impl EventProperties {
+    async fn from(data: Data) -> anyhow::Result<EventProperties> {
+        let stats = data.index_controller.get_all_stats().await?;
+
+        let database_size = stats.database_size;
+        let last_update_timestamp = stats.last_update.map(|u| u.timestamp());
+        let number_of_documents = stats
+            .indexes
+            .values()
+            .map(|index| index.number_of_documents)
+            .collect();
+
+        Ok(EventProperties {
+            database_size,
+            last_update_timestamp,
+            number_of_documents,
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct UserProperties<'a> {
+    env: &'a str,
+    start_since_days: u64,
+    user_email: Option<String>,
+    server_provider: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+struct Event<'a> {
+    user_id: &'a str,
+    event_type: &'a str,
+    device_id: &'a str,
+    time: u64,
+    app_version: &'a str,
+    user_properties: UserProperties<'a>,
+    event_properties: Option<EventProperties>,
+}
+
+#[derive(Debug, Serialize)]
+struct AmplitudeRequest<'a> {
+    api_key: &'a str,
+    events: Vec<Event<'a>>,
+}
+
+pub async fn analytics_sender(data: Data, opt: Opt) {
+    let username = whoami::username();
+    let hostname = whoami::hostname();
+    let platform = whoami::platform();
+
+    let uid = username + &hostname + &platform.to_string();
+
+    let mut hasher = SipHasher::new();
+    uid.hash(&mut hasher);
+    let hash = hasher.finish();
+
+    let uid = format!("{:X}", hash);
+    let platform = platform.to_string();
+    let first_start = Instant::now();
+
+    loop {
+        let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
+        let user_id = &uid;
+        let device_id = &platform;
+        let time = n.as_secs();
+        let event_type = "runtime_tick";
+        let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
+        let event_properties = EventProperties::from(data.clone()).await.ok();
+        let app_version = env!("CARGO_PKG_VERSION").to_string();
+        let app_version = app_version.as_str();
+        let user_email = std::env::var("MEILI_USER_EMAIL").ok();
+        let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok();
+        let user_properties = UserProperties {
+            env: &opt.env,
+            start_since_days: elapsed_since_start,
+            user_email,
+            server_provider,
+        };
+
+        let event = Event {
+            user_id,
+            event_type,
+            device_id,
+            time,
+            app_version,
+            user_properties,
+            event_properties,
+        };
+
+        let request = AmplitudeRequest {
+            api_key: AMPLITUDE_API_KEY,
+            events: vec![event],
+        };
+
+        let response = reqwest::Client::new()
+            .post("https://api2.amplitude.com/2/httpapi")
+            .timeout(Duration::from_secs(60)) // 1 minute max
+            .json(&request)
+            .send()
+            .await;
+        if let Err(e) = response {
+            debug!("Unsuccessful call to Amplitude: {}", e);
+        }
+
+        tokio::time::sleep(Duration::from_secs(3600)).await;
+    }
+}

meilisearch-http/src/data/mod.rs

@@ -0,0 +1,133 @@
+use std::ops::Deref;
+use std::sync::Arc;
+
+use sha2::Digest;
+
+use crate::index::{Checked, Settings};
+use crate::index_controller::{
+    error::Result, DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats,
+};
+use crate::option::Opt;
+
+pub mod search;
+mod updates;
+
+#[derive(Clone)]
+pub struct Data {
+    inner: Arc<DataInner>,
+}
+
+impl Deref for Data {
+    type Target = DataInner;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+pub struct DataInner {
+    pub index_controller: IndexController,
+    pub api_keys: ApiKeys,
+    options: Opt,
+}
+
+#[derive(Clone)]
+pub struct ApiKeys {
+    pub public: Option<String>,
+    pub private: Option<String>,
+    pub master: Option<String>,
+}
+
+impl ApiKeys {
+    pub fn generate_missing_api_keys(&mut self) {
+        if let Some(master_key) = &self.master {
+            if self.private.is_none() {
+                let key = format!("{}-private", master_key);
+                let sha = sha2::Sha256::digest(key.as_bytes());
+                self.private = Some(format!("{:x}", sha));
+            }
+            if self.public.is_none() {
+                let key = format!("{}-public", master_key);
+                let sha = sha2::Sha256::digest(key.as_bytes());
+                self.public = Some(format!("{:x}", sha));
+            }
+        }
+    }
+}
+
+impl Data {
+    pub fn new(options: Opt) -> anyhow::Result<Data> {
+        let path = options.db_path.clone();
+
+        let index_controller = IndexController::new(&path, &options)?;
+
+        let mut api_keys = ApiKeys {
+            master: options.clone().master_key,
+            private: None,
+            public: None,
+        };
+
+        api_keys.generate_missing_api_keys();
+
+        let inner = DataInner {
+            index_controller,
+            api_keys,
+            options,
+        };
+        let inner = Arc::new(inner);
+
+        Ok(Data { inner })
+    }
+
+    pub async fn settings(&self, uid: String) -> Result<Settings<Checked>> {
+        self.index_controller.settings(uid).await
+    }
+
+    pub async fn list_indexes(&self) -> Result<Vec<IndexMetadata>> {
+        self.index_controller.list_indexes().await
+    }
+
+    pub async fn index(&self, uid: String) -> Result<IndexMetadata> {
+        self.index_controller.get_index(uid).await
+    }
+
+    pub async fn create_index(
+        &self,
+        uid: String,
+        primary_key: Option<String>,
+    ) -> Result<IndexMetadata> {
+        let settings = IndexSettings {
+            uid: Some(uid),
+            primary_key,
+        };
+
+        let meta = self.index_controller.create_index(settings).await?;
+        Ok(meta)
+    }
+
+    pub async fn get_index_stats(&self, uid: String) -> Result<IndexStats> {
+        Ok(self.index_controller.get_index_stats(uid).await?)
+    }
+
+    pub async fn get_all_stats(&self) -> Result<Stats> {
+        Ok(self.index_controller.get_all_stats().await?)
+    }
+
+    pub async fn create_dump(&self) -> Result<DumpInfo> {
+        Ok(self.index_controller.create_dump().await?)
+    }
+
+    pub async fn dump_status(&self, uid: String) -> Result<DumpInfo> {
+        Ok(self.index_controller.dump_info(uid).await?)
+    }
+
+    #[inline]
+    pub fn http_payload_size_limit(&self) -> usize {
+        self.options.http_payload_size_limit.get_bytes() as usize
+    }
+
+    #[inline]
+    pub fn api_keys(&self) -> &ApiKeys {
+        &self.api_keys
+    }
+}

meilisearch-http/src/data/search.rs

@@ -0,0 +1,34 @@
+use serde_json::{Map, Value};
+
+use super::Data;
+use crate::index::{SearchQuery, SearchResult};
+use crate::index_controller::error::Result;
+
+impl Data {
+    pub async fn search(&self, index: String, search_query: SearchQuery) -> Result<SearchResult> {
+        self.index_controller.search(index, search_query).await
+    }
+
+    pub async fn retrieve_documents(
+        &self,
+        index: String,
+        offset: usize,
+        limit: usize,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Vec<Map<String, Value>>> {
+        self.index_controller
+            .documents(index, offset, limit, attributes_to_retrieve)
+            .await
+    }
+
+    pub async fn retrieve_document(
+        &self,
+        index: String,
+        document_id: String,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Map<String, Value>> {
+        self.index_controller
+            .document(index, document_id, attributes_to_retrieve)
+            .await
+    }
+}

meilisearch-http/src/data/updates.rs

@@ -0,0 +1,80 @@
+use milli::update::{IndexDocumentsMethod, UpdateFormat};
+
+use crate::extractors::payload::Payload;
+use crate::index::{Checked, Settings};
+use crate::index_controller::{error::Result, IndexMetadata, IndexSettings, UpdateStatus};
+use crate::Data;
+
+impl Data {
+    pub async fn add_documents(
+        &self,
+        index: String,
+        method: IndexDocumentsMethod,
+        format: UpdateFormat,
+        stream: Payload,
+        primary_key: Option<String>,
+    ) -> Result<UpdateStatus> {
+        let update_status = self
+            .index_controller
+            .add_documents(index, method, format, stream, primary_key)
+            .await?;
+        Ok(update_status)
+    }
+
+    pub async fn update_settings(
+        &self,
+        index: String,
+        settings: Settings<Checked>,
+        create: bool,
+    ) -> Result<UpdateStatus> {
+        let update = self
+            .index_controller
+            .update_settings(index, settings, create)
+            .await?;
+        Ok(update)
+    }
+
+    pub async fn clear_documents(&self, index: String) -> Result<UpdateStatus> {
+        let update = self.index_controller.clear_documents(index).await?;
+        Ok(update)
+    }
+
+    pub async fn delete_documents(
+        &self,
+        index: String,
+        document_ids: Vec<String>,
+    ) -> Result<UpdateStatus> {
+        let update = self
+            .index_controller
+            .delete_documents(index, document_ids)
+            .await?;
+        Ok(update)
+    }
+
+    pub async fn delete_index(&self, index: String) -> Result<()> {
+        self.index_controller.delete_index(index).await?;
+        Ok(())
+    }
+
+    pub async fn get_update_status(&self, index: String, uid: u64) -> Result<UpdateStatus> {
+        self.index_controller.update_status(index, uid).await
+    }
+
+    pub async fn get_updates_status(&self, index: String) -> Result<Vec<UpdateStatus>> {
+        self.index_controller.all_update_status(index).await
+    }
+
+    pub async fn update_index(
+        &self,
+        uid: String,
+        primary_key: Option<String>,
+        new_uid: Option<String>,
+    ) -> Result<IndexMetadata> {
+        let settings = IndexSettings {
+            uid: new_uid,
+            primary_key,
+        };
+
+        self.index_controller.update_index(uid, settings).await
+    }
+}

meilisearch-http/src/error.rs

@@ -0,0 +1,167 @@
+use std::error::Error;
+use std::fmt;
+
+use actix_web as aweb;
+use actix_web::body::Body;
+use actix_web::dev::BaseHttpResponseBuilder;
+use actix_web::http::StatusCode;
+use aweb::error::{JsonPayloadError, QueryPayloadError};
+use meilisearch_error::{Code, ErrorCode};
+use milli::UserError;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ResponseError {
+    #[serde(skip)]
+    code: StatusCode,
+    message: String,
+    error_code: String,
+    error_type: String,
+    error_link: String,
+}
+
+impl fmt::Display for ResponseError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.message.fmt(f)
+    }
+}
+
+impl<T> From<T> for ResponseError
+where
+    T: ErrorCode,
+{
+    fn from(other: T) -> Self {
+        Self {
+            code: other.http_status(),
+            message: other.to_string(),
+            error_code: other.error_name(),
+            error_type: other.error_type(),
+            error_link: other.error_url(),
+        }
+    }
+}
+
+impl aweb::error::ResponseError for ResponseError {
+    fn error_response(&self) -> aweb::BaseHttpResponse<Body> {
+        let json = serde_json::to_vec(self).unwrap();
+        BaseHttpResponseBuilder::new(self.status_code())
+            .content_type("application/json")
+            .body(json)
+    }
+
+    fn status_code(&self) -> StatusCode {
+        self.code
+    }
+}
+
+macro_rules! internal_error {
+    ($target:ty : $($other:path), *) => {
+        $(
+            impl From<$other> for $target {
+                fn from(other: $other) -> Self {
+                    Self::Internal(Box::new(other))
+                }
+            }
+        )*
+    }
+}
+
+#[derive(Debug)]
+pub struct MilliError<'a>(pub &'a milli::Error);
+
+impl Error for MilliError<'_> {}
+
+impl fmt::Display for MilliError<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl ErrorCode for MilliError<'_> {
+    fn error_code(&self) -> Code {
+        match self.0 {
+            milli::Error::InternalError(_) => Code::Internal,
+            milli::Error::IoError(_) => Code::Internal,
+            milli::Error::UserError(ref error) => {
+                match error {
+                    // TODO: wait for spec for new error codes.
+                    UserError::Csv(_)
+                    | UserError::SerdeJson(_)
+                    | UserError::MaxDatabaseSizeReached
+                    | UserError::InvalidCriterionName { .. }
+                    | UserError::InvalidDocumentId { .. }
+                    | UserError::InvalidStoreFile
+                    | UserError::NoSpaceLeftOnDevice
+                    | UserError::DocumentLimitReached => Code::Internal,
+                    UserError::AttributeLimitReached => Code::MaxFieldsLimitExceeded,
+                    UserError::InvalidFilter(_) => Code::Filter,
+                    UserError::InvalidFilterAttribute(_) => Code::Filter,
+                    UserError::MissingDocumentId { .. } => Code::MissingDocumentId,
+                    UserError::MissingPrimaryKey => Code::MissingPrimaryKey,
+                    UserError::PrimaryKeyCannotBeChanged => Code::PrimaryKeyAlreadyPresent,
+                    UserError::PrimaryKeyCannotBeReset => Code::PrimaryKeyAlreadyPresent,
+                    UserError::UnknownInternalDocumentId { .. } => Code::DocumentNotFound,
+                    UserError::InvalidFacetsDistribution { .. } => Code::BadRequest,
+                }
+            }
+        }
+    }
+}
+
+impl fmt::Display for PayloadError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            PayloadError::Json(e) => e.fmt(f),
+            PayloadError::Query(e) => e.fmt(f),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum PayloadError {
+    Json(JsonPayloadError),
+    Query(QueryPayloadError),
+}
+
+impl Error for PayloadError {}
+
+impl ErrorCode for PayloadError {
+    fn error_code(&self) -> Code {
+        match self {
+            PayloadError::Json(err) => match err {
+                JsonPayloadError::Overflow => Code::PayloadTooLarge,
+                JsonPayloadError::ContentType => Code::UnsupportedMediaType,
+                JsonPayloadError::Payload(aweb::error::PayloadError::Overflow) => {
+                    Code::PayloadTooLarge
+                }
+                JsonPayloadError::Deserialize(_) | JsonPayloadError::Payload(_) => Code::BadRequest,
+                JsonPayloadError::Serialize(_) => Code::Internal,
+                _ => Code::Internal,
+            },
+            PayloadError::Query(err) => match err {
+                QueryPayloadError::Deserialize(_) => Code::BadRequest,
+                _ => Code::Internal,
+            },
+        }
+    }
+}
+
+impl From<JsonPayloadError> for PayloadError {
+    fn from(other: JsonPayloadError) -> Self {
+        Self::Json(other)
+    }
+}
+
+impl From<QueryPayloadError> for PayloadError {
+    fn from(other: QueryPayloadError) -> Self {
+        Self::Query(other)
+    }
+}
+
+pub fn payload_error_handler<E>(err: E) -> ResponseError
+where
+    E: Into<PayloadError>,
+{
+    err.into().into()
+}

meilisearch-http/src/extractors/authentication/error.rs

@@ -0,0 +1,25 @@
+use meilisearch_error::{Code, ErrorCode};
+
+#[derive(Debug, thiserror::Error)]
+pub enum AuthenticationError {
+    #[error("You must have an authorization token")]
+    MissingAuthorizationHeader,
+    #[error("Invalid API key")]
+    InvalidToken(String),
+    // Triggered on configuration error.
+    #[error("Irretrievable state")]
+    IrretrievableState,
+    #[error("Unknown authentication policy")]
+    UnknownPolicy,
+}
+
+impl ErrorCode for AuthenticationError {
+    fn error_code(&self) -> Code {
+        match self {
+            AuthenticationError::MissingAuthorizationHeader => Code::MissingAuthorizationHeader,
+            AuthenticationError::InvalidToken(_) => Code::InvalidToken,
+            AuthenticationError::IrretrievableState => Code::Internal,
+            AuthenticationError::UnknownPolicy => Code::Internal,
+        }
+    }
+}

meilisearch-http/src/extractors/authentication/mod.rs

@@ -0,0 +1,182 @@
+mod error;
+
+use std::any::{Any, TypeId};
+use std::collections::HashMap;
+use std::marker::PhantomData;
+use std::ops::Deref;
+
+use actix_web::FromRequest;
+use futures::future::err;
+use futures::future::{ok, Ready};
+
+use crate::error::ResponseError;
+use error::AuthenticationError;
+
+macro_rules! create_policies {
+    ($($name:ident), *) => {
+        pub mod policies {
+            use std::collections::HashSet;
+            use crate::extractors::authentication::Policy;
+
+            $(
+                #[derive(Debug, Default)]
+                pub struct $name {
+                    inner: HashSet<Vec<u8>>
+                }
+
+                impl $name {
+                    pub fn new() -> Self {
+                        Self { inner: HashSet::new() }
+                    }
+
+                    pub fn add(&mut self, token: Vec<u8>) {
+                        self.inner.insert(token);
+                    }
+                }
+
+                impl Policy for $name {
+                    fn authenticate(&self, token: &[u8]) -> bool {
+                        self.inner.contains(token)
+                    }
+                }
+            )*
+        }
+    };
+}
+
+create_policies!(Public, Private, Admin);
+
+/// Instanciate a `Policies`, filled with the given policies.
+macro_rules! init_policies {
+    ($($name:ident), *) => {
+        {
+            let mut policies = crate::extractors::authentication::Policies::new();
+            $(
+                let policy = $name::new();
+                policies.insert(policy);
+            )*
+            policies
+        }
+    };
+}
+
+/// Adds user to all specified policies.
+macro_rules! create_users {
+    ($policies:ident, $($user:expr => { $($policy:ty), * }), *) => {
+        {
+            $(
+                $(
+                    $policies.get_mut::<$policy>().map(|p| p.add($user.to_owned()));
+                )*
+            )*
+        }
+    };
+}
+
+pub struct GuardedData<T, D> {
+    data: D,
+    _marker: PhantomData<T>,
+}
+
+impl<T, D> Deref for GuardedData<T, D> {
+    type Target = D;
+
+    fn deref(&self) -> &Self::Target {
+        &self.data
+    }
+}
+
+pub trait Policy {
+    fn authenticate(&self, token: &[u8]) -> bool;
+}
+
+#[derive(Debug)]
+pub struct Policies {
+    inner: HashMap<TypeId, Box<dyn Any>>,
+}
+
+impl Policies {
+    pub fn new() -> Self {
+        Self {
+            inner: HashMap::new(),
+        }
+    }
+
+    pub fn insert<S: Policy + 'static>(&mut self, policy: S) {
+        self.inner.insert(TypeId::of::<S>(), Box::new(policy));
+    }
+
+    pub fn get<S: Policy + 'static>(&self) -> Option<&S> {
+        self.inner
+            .get(&TypeId::of::<S>())
+            .and_then(|p| p.downcast_ref::<S>())
+    }
+
+    pub fn get_mut<S: Policy + 'static>(&mut self) -> Option<&mut S> {
+        self.inner
+            .get_mut(&TypeId::of::<S>())
+            .and_then(|p| p.downcast_mut::<S>())
+    }
+}
+
+impl Default for Policies {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+pub enum AuthConfig {
+    NoAuth,
+    Auth(Policies),
+}
+
+impl Default for AuthConfig {
+    fn default() -> Self {
+        Self::NoAuth
+    }
+}
+
+impl<P: Policy + 'static, D: 'static + Clone> FromRequest for GuardedData<P, D> {
+    type Config = AuthConfig;
+
+    type Error = ResponseError;
+
+    type Future = Ready<Result<Self, Self::Error>>;
+
+    fn from_request(
+        req: &actix_web::HttpRequest,
+        _payload: &mut actix_http::Payload,
+    ) -> Self::Future {
+        match req.app_data::<Self::Config>() {
+            Some(config) => match config {
+                AuthConfig::NoAuth => match req.app_data::<D>().cloned() {
+                    Some(data) => ok(Self {
+                        data,
+                        _marker: PhantomData,
+                    }),
+                    None => err(AuthenticationError::IrretrievableState.into()),
+                },
+                AuthConfig::Auth(policies) => match policies.get::<P>() {
+                    Some(policy) => match req.headers().get("x-meili-api-key") {
+                        Some(token) => {
+                            if policy.authenticate(token.as_bytes()) {
+                                match req.app_data::<D>().cloned() {
+                                    Some(data) => ok(Self {
+                                        data,
+                                        _marker: PhantomData,
+                                    }),
+                                    None => err(AuthenticationError::IrretrievableState.into()),
+                                }
+                            } else {
+                                err(AuthenticationError::InvalidToken(String::from("hello")).into())
+                            }
+                        }
+                        None => err(AuthenticationError::MissingAuthorizationHeader.into()),
+                    },
+                    None => err(AuthenticationError::UnknownPolicy.into()),
+                },
+            },
+            None => err(AuthenticationError::IrretrievableState.into()),
+        }
+    }
+}

meilisearch-http/src/extractors/mod.rs

@@ -0,0 +1,3 @@
+pub mod payload;
+#[macro_use]
+pub mod authentication;

meilisearch-http/src/extractors/payload.rs

@@ -0,0 +1,69 @@
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use actix_http::error::PayloadError;
+use actix_web::{dev, web, FromRequest, HttpRequest};
+use futures::future::{ready, Ready};
+use futures::Stream;
+
+pub struct Payload {
+    payload: dev::Payload,
+    limit: usize,
+}
+
+pub struct PayloadConfig {
+    limit: usize,
+}
+
+impl PayloadConfig {
+    pub fn new(limit: usize) -> Self {
+        Self { limit }
+    }
+}
+
+impl Default for PayloadConfig {
+    fn default() -> Self {
+        Self { limit: 256 * 1024 }
+    }
+}
+
+impl FromRequest for Payload {
+    type Config = PayloadConfig;
+
+    type Error = PayloadError;
+
+    type Future = Ready<Result<Payload, Self::Error>>;
+
+    #[inline]
+    fn from_request(req: &HttpRequest, payload: &mut dev::Payload) -> Self::Future {
+        let limit = req
+            .app_data::<PayloadConfig>()
+            .map(|c| c.limit)
+            .unwrap_or(Self::Config::default().limit);
+        ready(Ok(Payload {
+            payload: payload.take(),
+            limit,
+        }))
+    }
+}
+
+impl Stream for Payload {
+    type Item = Result<web::Bytes, PayloadError>;
+
+    #[inline]
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        match Pin::new(&mut self.payload).poll_next(cx) {
+            Poll::Ready(Some(result)) => match result {
+                Ok(bytes) => match self.limit.checked_sub(bytes.len()) {
+                    Some(new_limit) => {
+                        self.limit = new_limit;
+                        Poll::Ready(Some(Ok(bytes)))
+                    }
+                    None => Poll::Ready(Some(Err(PayloadError::Overflow))),
+                },
+                x => Poll::Ready(Some(x)),
+            },
+            otherwise => otherwise,
+        }
+    }
+}

meilisearch-http/src/helpers/compression.rs

@@ -0,0 +1,26 @@
+use std::fs::{create_dir_all, File};
+use std::io::Write;
+use std::path::Path;
+
+use flate2::{read::GzDecoder, write::GzEncoder, Compression};
+use tar::{Archive, Builder};
+
+pub fn to_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Result<()> {
+    let mut f = File::create(dest)?;
+    let gz_encoder = GzEncoder::new(&mut f, Compression::default());
+    let mut tar_encoder = Builder::new(gz_encoder);
+    tar_encoder.append_dir_all(".", src)?;
+    let gz_encoder = tar_encoder.into_inner()?;
+    gz_encoder.finish()?;
+    f.flush()?;
+    Ok(())
+}
+
+pub fn from_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Result<()> {
+    let f = File::open(&src)?;
+    let gz = GzDecoder::new(f);
+    let mut ar = Archive::new(gz);
+    create_dir_all(&dest)?;
+    ar.unpack(&dest)?;
+    Ok(())
+}

meilisearch-http/src/helpers/env.rs

@@ -0,0 +1,16 @@
+use walkdir::WalkDir;
+
+pub trait EnvSizer {
+    fn size(&self) -> u64;
+}
+
+impl EnvSizer for heed::Env {
+    fn size(&self) -> u64 {
+        WalkDir::new(self.path())
+            .into_iter()
+            .filter_map(|entry| entry.ok())
+            .filter_map(|entry| entry.metadata().ok())
+            .filter(|metadata| metadata.is_file())
+            .fold(0, |acc, m| acc + m.len())
+    }
+}

meilisearch-http/src/helpers/mod.rs

@@ -0,0 +1,4 @@
+pub mod compression;
+mod env;
+
+pub use env::EnvSizer;

meilisearch-http/src/index/dump.rs

@@ -0,0 +1,134 @@
+use std::fs::{create_dir_all, File};
+use std::io::{BufRead, BufReader, Write};
+use std::path::Path;
+use std::sync::Arc;
+
+use anyhow::{bail, Context};
+use heed::RoTxn;
+use indexmap::IndexMap;
+use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream};
+use serde::{Deserialize, Serialize};
+
+use crate::option::IndexerOpts;
+
+use super::error::Result;
+use super::{update_handler::UpdateHandler, Index, Settings, Unchecked};
+
+#[derive(Serialize, Deserialize)]
+struct DumpMeta {
+    settings: Settings<Unchecked>,
+    primary_key: Option<String>,
+}
+
+const META_FILE_NAME: &str = "meta.json";
+const DATA_FILE_NAME: &str = "documents.jsonl";
+
+impl Index {
+    pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
+        // acquire write txn make sure any ongoing write is finished before we start.
+        let txn = self.env.write_txn()?;
+
+        self.dump_documents(&txn, &path)?;
+        self.dump_meta(&txn, &path)?;
+
+        Ok(())
+    }
+
+    fn dump_documents(&self, txn: &RoTxn, path: impl AsRef<Path>) -> Result<()> {
+        let document_file_path = path.as_ref().join(DATA_FILE_NAME);
+        let mut document_file = File::create(&document_file_path)?;
+
+        let documents = self.all_documents(txn)?;
+        let fields_ids_map = self.fields_ids_map(txn)?;
+
+        // dump documents
+        let mut json_map = IndexMap::new();
+        for document in documents {
+            let (_, reader) = document?;
+
+            for (fid, bytes) in reader.iter() {
+                if let Some(name) = fields_ids_map.name(fid) {
+                    json_map.insert(name, serde_json::from_slice::<serde_json::Value>(bytes)?);
+                }
+            }
+
+            serde_json::to_writer(&mut document_file, &json_map)?;
+            document_file.write_all(b"\n")?;
+
+            json_map.clear();
+        }
+
+        Ok(())
+    }
+
+    fn dump_meta(&self, txn: &RoTxn, path: impl AsRef<Path>) -> Result<()> {
+        let meta_file_path = path.as_ref().join(META_FILE_NAME);
+        let mut meta_file = File::create(&meta_file_path)?;
+
+        let settings = self.settings_txn(txn)?.into_unchecked();
+        let primary_key = self.primary_key(txn)?.map(String::from);
+        let meta = DumpMeta {
+            settings,
+            primary_key,
+        };
+
+        serde_json::to_writer(&mut meta_file, &meta)?;
+
+        Ok(())
+    }
+
+    pub fn load_dump(
+        src: impl AsRef<Path>,
+        dst: impl AsRef<Path>,
+        size: usize,
+        indexing_options: &IndexerOpts,
+    ) -> anyhow::Result<()> {
+        let dir_name = src
+            .as_ref()
+            .file_name()
+            .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?;
+
+        let dst_dir_path = dst.as_ref().join("indexes").join(dir_name);
+        create_dir_all(&dst_dir_path)?;
+
+        let meta_path = src.as_ref().join(META_FILE_NAME);
+        let mut meta_file = File::open(meta_path)?;
+        let DumpMeta {
+            settings,
+            primary_key,
+        } = serde_json::from_reader(&mut meta_file)?;
+        let settings = settings.check();
+        let index = Self::open(&dst_dir_path, size)?;
+        let mut txn = index.write_txn()?;
+
+        let handler = UpdateHandler::new(&indexing_options)?;
+
+        index.update_settings_txn(&mut txn, &settings, handler.update_builder(0))?;
+
+        let document_file_path = src.as_ref().join(DATA_FILE_NAME);
+        let reader = File::open(&document_file_path)?;
+        let mut reader = BufReader::new(reader);
+        reader.fill_buf()?;
+        // If the document file is empty, we don't perform the document addition, to prevent
+        // a primary key error to be thrown.
+        if !reader.buffer().is_empty() {
+            index.update_documents_txn(
+                &mut txn,
+                JsonStream,
+                IndexDocumentsMethod::UpdateDocuments,
+                Some(reader),
+                handler.update_builder(0),
+                primary_key.as_deref(),
+            )?;
+        }
+
+        txn.commit()?;
+
+        match Arc::try_unwrap(index.0) {
+            Ok(inner) => inner.prepare_for_closing().wait(),
+            Err(_) => bail!("Could not close index properly."),
+        }
+
+        Ok(())
+    }
+}

meilisearch-http/src/index/error.rs

@@ -0,0 +1,52 @@
+use std::error::Error;
+
+use meilisearch_error::{Code, ErrorCode};
+use serde_json::Value;
+
+use crate::error::MilliError;
+
+pub type Result<T> = std::result::Result<T, IndexError>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum IndexError {
+    #[error("Internal error: {0}")]
+    Internal(Box<dyn Error + Send + Sync + 'static>),
+    #[error("Document with id {0} not found.")]
+    DocumentNotFound(String),
+    #[error("{0}")]
+    Facet(#[from] FacetError),
+    #[error("{0}")]
+    Milli(#[from] milli::Error),
+}
+
+internal_error!(
+    IndexError: std::io::Error,
+    heed::Error,
+    fst::Error,
+    serde_json::Error
+);
+
+impl ErrorCode for IndexError {
+    fn error_code(&self) -> Code {
+        match self {
+            IndexError::Internal(_) => Code::Internal,
+            IndexError::DocumentNotFound(_) => Code::DocumentNotFound,
+            IndexError::Facet(e) => e.error_code(),
+            IndexError::Milli(e) => MilliError(e).error_code(),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum FacetError {
+    #[error("Invalid facet expression, expected {}, found: {1}", .0.join(", "))]
+    InvalidExpression(&'static [&'static str], Value),
+}
+
+impl ErrorCode for FacetError {
+    fn error_code(&self) -> Code {
+        match self {
+            FacetError::InvalidExpression(_, _) => Code::Facet,
+        }
+    }
+}

meilisearch-http/src/index/mod.rs

@@ -0,0 +1,194 @@
+use std::collections::{BTreeSet, HashSet};
+use std::fs::create_dir_all;
+use std::marker::PhantomData;
+use std::ops::Deref;
+use std::path::Path;
+use std::sync::Arc;
+
+use heed::{EnvOpenOptions, RoTxn};
+use milli::obkv_to_json;
+use serde::{de::Deserializer, Deserialize};
+use serde_json::{Map, Value};
+
+use crate::helpers::EnvSizer;
+use error::Result;
+
+pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT};
+pub use updates::{Checked, Facets, Settings, Unchecked};
+
+use self::error::IndexError;
+
+pub mod error;
+pub mod update_handler;
+
+mod dump;
+mod search;
+mod updates;
+
+pub type Document = Map<String, Value>;
+
+#[derive(Clone)]
+pub struct Index(pub Arc<milli::Index>);
+
+impl Deref for Index {
+    type Target = milli::Index;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.as_ref()
+    }
+}
+
+pub fn deserialize_some<'de, T, D>(deserializer: D) -> std::result::Result<Option<T>, D::Error>
+where
+    T: Deserialize<'de>,
+    D: Deserializer<'de>,
+{
+    Deserialize::deserialize(deserializer).map(Some)
+}
+
+impl Index {
+    pub fn open(path: impl AsRef<Path>, size: usize) -> Result<Self> {
+        create_dir_all(&path)?;
+        let mut options = EnvOpenOptions::new();
+        options.map_size(size);
+        let index = milli::Index::new(options, &path)?;
+        Ok(Index(Arc::new(index)))
+    }
+
+    pub fn settings(&self) -> Result<Settings<Checked>> {
+        let txn = self.read_txn()?;
+        self.settings_txn(&txn)
+    }
+
+    pub fn settings_txn(&self, txn: &RoTxn) -> Result<Settings<Checked>> {
+        let displayed_attributes = self
+            .displayed_fields(&txn)?
+            .map(|fields| fields.into_iter().map(String::from).collect());
+
+        let searchable_attributes = self
+            .searchable_fields(&txn)?
+            .map(|fields| fields.into_iter().map(String::from).collect());
+
+        let faceted_attributes = self.faceted_fields(&txn)?.into_iter().collect();
+
+        let criteria = self
+            .criteria(&txn)?
+            .into_iter()
+            .map(|c| c.to_string())
+            .collect();
+
+        let stop_words = self
+            .stop_words(&txn)?
+            .map(|stop_words| -> Result<BTreeSet<_>> {
+                Ok(stop_words.stream().into_strs()?.into_iter().collect())
+            })
+            .transpose()?
+            .unwrap_or_else(BTreeSet::new);
+        let distinct_field = self.distinct_field(&txn)?.map(String::from);
+
+        // in milli each word in the synonyms map were split on their separator. Since we lost
+        // this information we are going to put space between words.
+        let synonyms = self
+            .synonyms(&txn)?
+            .iter()
+            .map(|(key, values)| {
+                (
+                    key.join(" "),
+                    values.iter().map(|value| value.join(" ")).collect(),
+                )
+            })
+            .collect();
+
+        Ok(Settings {
+            displayed_attributes: Some(displayed_attributes),
+            searchable_attributes: Some(searchable_attributes),
+            filterable_attributes: Some(Some(faceted_attributes)),
+            ranking_rules: Some(Some(criteria)),
+            stop_words: Some(Some(stop_words)),
+            distinct_attribute: Some(distinct_field),
+            synonyms: Some(Some(synonyms)),
+            _kind: PhantomData,
+        })
+    }
+
+    pub fn retrieve_documents<S: AsRef<str>>(
+        &self,
+        offset: usize,
+        limit: usize,
+        attributes_to_retrieve: Option<Vec<S>>,
+    ) -> Result<Vec<Map<String, Value>>> {
+        let txn = self.read_txn()?;
+
+        let fields_ids_map = self.fields_ids_map(&txn)?;
+        let fields_to_display =
+            self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?;
+
+        let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit);
+
+        let mut documents = Vec::new();
+
+        for entry in iter {
+            let (_id, obkv) = entry?;
+            let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?;
+            documents.push(object);
+        }
+
+        Ok(documents)
+    }
+
+    pub fn retrieve_document<S: AsRef<str>>(
+        &self,
+        doc_id: String,
+        attributes_to_retrieve: Option<Vec<S>>,
+    ) -> Result<Map<String, Value>> {
+        let txn = self.read_txn()?;
+
+        let fields_ids_map = self.fields_ids_map(&txn)?;
+
+        let fields_to_display =
+            self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?;
+
+        let internal_id = self
+            .external_documents_ids(&txn)?
+            .get(doc_id.as_bytes())
+            .ok_or_else(|| IndexError::DocumentNotFound(doc_id.clone()))?;
+
+        let document = self
+            .documents(&txn, std::iter::once(internal_id))?
+            .into_iter()
+            .next()
+            .map(|(_, d)| d)
+            .ok_or(IndexError::DocumentNotFound(doc_id))?;
+
+        let document = obkv_to_json(&fields_to_display, &fields_ids_map, document)?;
+
+        Ok(document)
+    }
+
+    pub fn size(&self) -> u64 {
+        self.env.size()
+    }
+
+    fn fields_to_display<S: AsRef<str>>(
+        &self,
+        txn: &heed::RoTxn,
+        attributes_to_retrieve: &Option<Vec<S>>,
+        fields_ids_map: &milli::FieldsIdsMap,
+    ) -> Result<Vec<u8>> {
+        let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? {
+            Some(ids) => ids.into_iter().collect::<Vec<_>>(),
+            None => fields_ids_map.iter().map(|(id, _)| id).collect(),
+        };
+
+        let attributes_to_retrieve_ids = match attributes_to_retrieve {
+            Some(attrs) => attrs
+                .iter()
+                .filter_map(|f| fields_ids_map.id(f.as_ref()))
+                .collect::<HashSet<_>>(),
+            None => fields_ids_map.iter().map(|(id, _)| id).collect(),
+        };
+
+        displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid));
+        Ok(displayed_fields_ids)
+    }
+}

meilisearch-http/src/index/update_handler.rs

@@ -0,0 +1,92 @@
+use std::fs::File;
+
+use crate::index::Index;
+use grenad::CompressionType;
+use milli::update::UpdateBuilder;
+use rayon::ThreadPool;
+
+use crate::index_controller::UpdateMeta;
+use crate::index_controller::{Failed, Processed, Processing};
+use crate::option::IndexerOpts;
+
+pub struct UpdateHandler {
+    max_nb_chunks: Option<usize>,
+    chunk_compression_level: Option<u32>,
+    thread_pool: ThreadPool,
+    log_frequency: usize,
+    max_memory: usize,
+    linked_hash_map_size: usize,
+    chunk_compression_type: CompressionType,
+    chunk_fusing_shrink_size: u64,
+}
+
+impl UpdateHandler {
+    pub fn new(opt: &IndexerOpts) -> anyhow::Result<Self> {
+        let thread_pool = rayon::ThreadPoolBuilder::new()
+            .num_threads(opt.indexing_jobs.unwrap_or(num_cpus::get() / 2))
+            .build()?;
+        Ok(Self {
+            max_nb_chunks: opt.max_nb_chunks,
+            chunk_compression_level: opt.chunk_compression_level,
+            thread_pool,
+            log_frequency: opt.log_every_n,
+            max_memory: opt.max_memory.get_bytes() as usize,
+            linked_hash_map_size: opt.linked_hash_map_size,
+            chunk_compression_type: opt.chunk_compression_type,
+            chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(),
+        })
+    }
+
+    pub fn update_builder(&self, update_id: u64) -> UpdateBuilder {
+        // We prepare the update by using the update builder.
+        let mut update_builder = UpdateBuilder::new(update_id);
+        if let Some(max_nb_chunks) = self.max_nb_chunks {
+            update_builder.max_nb_chunks(max_nb_chunks);
+        }
+        if let Some(chunk_compression_level) = self.chunk_compression_level {
+            update_builder.chunk_compression_level(chunk_compression_level);
+        }
+        update_builder.thread_pool(&self.thread_pool);
+        update_builder.log_every_n(self.log_frequency);
+        update_builder.max_memory(self.max_memory);
+        update_builder.linked_hash_map_size(self.linked_hash_map_size);
+        update_builder.chunk_compression_type(self.chunk_compression_type);
+        update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size);
+        update_builder
+    }
+
+    pub fn handle_update(
+        &self,
+        meta: Processing,
+        content: Option<File>,
+        index: Index,
+    ) -> Result<Processed, Failed> {
+        use UpdateMeta::*;
+
+        let update_id = meta.id();
+
+        let update_builder = self.update_builder(update_id);
+
+        let result = match meta.meta() {
+            DocumentsAddition {
+                method,
+                format,
+                primary_key,
+            } => index.update_documents(
+                *format,
+                *method,
+                content,
+                update_builder,
+                primary_key.as_deref(),
+            ),
+            ClearDocuments => index.clear_documents(update_builder),
+            DeleteDocuments { ids } => index.delete_documents(ids, update_builder),
+            Settings(settings) => index.update_settings(&settings.clone().check(), update_builder),
+        };
+
+        match result {
+            Ok(result) => Ok(meta.process(result)),
+            Err(e) => Err(meta.fail(e.into())),
+        }
+    }
+}

meilisearch-http/src/index/updates.rs

@@ -0,0 +1,382 @@
+use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::io;
+use std::marker::PhantomData;
+use std::num::NonZeroUsize;
+
+use flate2::read::GzDecoder;
+use log::{debug, info, trace};
+use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat};
+use serde::{Deserialize, Serialize, Serializer};
+
+use crate::index_controller::UpdateResult;
+
+use super::error::Result;
+use super::{deserialize_some, Index};
+
+fn serialize_with_wildcard<S>(
+    field: &Option<Option<Vec<String>>>,
+    s: S,
+) -> std::result::Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let wildcard = vec!["*".to_string()];
+    s.serialize_some(&field.as_ref().map(|o| o.as_ref().unwrap_or(&wildcard)))
+}
+
+#[derive(Clone, Default, Debug, Serialize)]
+pub struct Checked;
+#[derive(Clone, Default, Debug, Serialize, Deserialize)]
+pub struct Unchecked;
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+#[serde(rename_all = "camelCase")]
+#[serde(bound(serialize = "T: Serialize", deserialize = "T: Deserialize<'static>"))]
+pub struct Settings<T> {
+    #[serde(
+        default,
+        deserialize_with = "deserialize_some",
+        serialize_with = "serialize_with_wildcard",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub displayed_attributes: Option<Option<Vec<String>>>,
+
+    #[serde(
+        default,
+        deserialize_with = "deserialize_some",
+        serialize_with = "serialize_with_wildcard",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub searchable_attributes: Option<Option<Vec<String>>>,
+
+    #[serde(
+        default,
+        deserialize_with = "deserialize_some",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub filterable_attributes: Option<Option<HashSet<String>>>,
+
+    #[serde(
+        default,
+        deserialize_with = "deserialize_some",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub ranking_rules: Option<Option<Vec<String>>>,
+    #[serde(
+        default,
+        deserialize_with = "deserialize_some",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub stop_words: Option<Option<BTreeSet<String>>>,
+    #[serde(
+        default,
+        deserialize_with = "deserialize_some",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
+    #[serde(
+        default,
+        deserialize_with = "deserialize_some",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub distinct_attribute: Option<Option<String>>,
+
+    #[serde(skip)]
+    pub _kind: PhantomData<T>,
+}
+
+impl Settings<Checked> {
+    pub fn cleared() -> Settings<Checked> {
+        Settings {
+            displayed_attributes: Some(None),
+            searchable_attributes: Some(None),
+            filterable_attributes: Some(None),
+            ranking_rules: Some(None),
+            stop_words: Some(None),
+            synonyms: Some(None),
+            distinct_attribute: Some(None),
+            _kind: PhantomData,
+        }
+    }
+
+    pub fn into_unchecked(self) -> Settings<Unchecked> {
+        let Self {
+            displayed_attributes,
+            searchable_attributes,
+            filterable_attributes,
+            ranking_rules,
+            stop_words,
+            synonyms,
+            distinct_attribute,
+            ..
+        } = self;
+
+        Settings {
+            displayed_attributes,
+            searchable_attributes,
+            filterable_attributes,
+            ranking_rules,
+            stop_words,
+            synonyms,
+            distinct_attribute,
+            _kind: PhantomData,
+        }
+    }
+}
+
+impl Settings<Unchecked> {
+    pub fn check(mut self) -> Settings<Checked> {
+        let displayed_attributes = match self.displayed_attributes.take() {
+            Some(Some(fields)) => {
+                if fields.iter().any(|f| f == "*") {
+                    Some(None)
+                } else {
+                    Some(Some(fields))
+                }
+            }
+            otherwise => otherwise,
+        };
+
+        let searchable_attributes = match self.searchable_attributes.take() {
+            Some(Some(fields)) => {
+                if fields.iter().any(|f| f == "*") {
+                    Some(None)
+                } else {
+                    Some(Some(fields))
+                }
+            }
+            otherwise => otherwise,
+        };
+
+        Settings {
+            displayed_attributes,
+            searchable_attributes,
+            filterable_attributes: self.filterable_attributes,
+            ranking_rules: self.ranking_rules,
+            stop_words: self.stop_words,
+            synonyms: self.synonyms,
+            distinct_attribute: self.distinct_attribute,
+            _kind: PhantomData,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+#[serde(rename_all = "camelCase")]
+pub struct Facets {
+    pub level_group_size: Option<NonZeroUsize>,
+    pub min_level_size: Option<NonZeroUsize>,
+}
+
+impl Index {
+    pub fn update_documents(
+        &self,
+        format: UpdateFormat,
+        method: IndexDocumentsMethod,
+        content: Option<impl io::Read>,
+        update_builder: UpdateBuilder,
+        primary_key: Option<&str>,
+    ) -> Result<UpdateResult> {
+        let mut txn = self.write_txn()?;
+        let result = self.update_documents_txn(
+            &mut txn,
+            format,
+            method,
+            content,
+            update_builder,
+            primary_key,
+        )?;
+        txn.commit()?;
+        Ok(result)
+    }
+
+    pub fn update_documents_txn<'a, 'b>(
+        &'a self,
+        txn: &mut heed::RwTxn<'a, 'b>,
+        format: UpdateFormat,
+        method: IndexDocumentsMethod,
+        content: Option<impl io::Read>,
+        update_builder: UpdateBuilder,
+        primary_key: Option<&str>,
+    ) -> Result<UpdateResult> {
+        trace!("performing document addition");
+
+        // Set the primary key if not set already, ignore if already set.
+        if let (None, Some(primary_key)) = (self.primary_key(txn)?, primary_key) {
+            let mut builder = UpdateBuilder::new(0).settings(txn, &self);
+            builder.set_primary_key(primary_key.to_string());
+            builder.execute(|_, _| ())?;
+        }
+
+        let mut builder = update_builder.index_documents(txn, self);
+        builder.update_format(format);
+        builder.index_documents_method(method);
+
+        let indexing_callback =
+            |indexing_step, update_id| debug!("update {}: {:?}", update_id, indexing_step);
+
+        let gzipped = false;
+        let addition = match content {
+            Some(content) if gzipped => {
+                builder.execute(GzDecoder::new(content), indexing_callback)?
+            }
+            Some(content) => builder.execute(content, indexing_callback)?,
+            None => builder.execute(std::io::empty(), indexing_callback)?,
+        };
+
+        info!("document addition done: {:?}", addition);
+
+        Ok(UpdateResult::DocumentsAddition(addition))
+    }
+
+    pub fn clear_documents(&self, update_builder: UpdateBuilder) -> Result<UpdateResult> {
+        // We must use the write transaction of the update here.
+        let mut wtxn = self.write_txn()?;
+        let builder = update_builder.clear_documents(&mut wtxn, self);
+
+        let _count = builder.execute()?;
+
+        wtxn.commit()
+            .and(Ok(UpdateResult::Other))
+            .map_err(Into::into)
+    }
+
+    pub fn update_settings_txn<'a, 'b>(
+        &'a self,
+        txn: &mut heed::RwTxn<'a, 'b>,
+        settings: &Settings<Checked>,
+        update_builder: UpdateBuilder,
+    ) -> Result<UpdateResult> {
+        // We must use the write transaction of the update here.
+        let mut builder = update_builder.settings(txn, self);
+
+        if let Some(ref names) = settings.searchable_attributes {
+            match names {
+                Some(names) => builder.set_searchable_fields(names.clone()),
+                None => builder.reset_searchable_fields(),
+            }
+        }
+
+        if let Some(ref names) = settings.displayed_attributes {
+            match names {
+                Some(names) => builder.set_displayed_fields(names.clone()),
+                None => builder.reset_displayed_fields(),
+            }
+        }
+
+        if let Some(ref facet_types) = settings.filterable_attributes {
+            let facet_types = facet_types.clone().unwrap_or_else(HashSet::new);
+            builder.set_filterable_fields(facet_types);
+        }
+
+        if let Some(ref criteria) = settings.ranking_rules {
+            match criteria {
+                Some(criteria) => builder.set_criteria(criteria.clone()),
+                None => builder.reset_criteria(),
+            }
+        }
+
+        if let Some(ref stop_words) = settings.stop_words {
+            match stop_words {
+                Some(stop_words) => builder.set_stop_words(stop_words.clone()),
+                None => builder.reset_stop_words(),
+            }
+        }
+
+        if let Some(ref synonyms) = settings.synonyms {
+            match synonyms {
+                Some(synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()),
+                None => builder.reset_synonyms(),
+            }
+        }
+
+        if let Some(ref distinct_attribute) = settings.distinct_attribute {
+            match distinct_attribute {
+                Some(attr) => builder.set_distinct_field(attr.clone()),
+                None => builder.reset_distinct_field(),
+            }
+        }
+
+        builder.execute(|indexing_step, update_id| {
+            debug!("update {}: {:?}", update_id, indexing_step)
+        })?;
+
+        Ok(UpdateResult::Other)
+    }
+
+    pub fn update_settings(
+        &self,
+        settings: &Settings<Checked>,
+        update_builder: UpdateBuilder,
+    ) -> Result<UpdateResult> {
+        let mut txn = self.write_txn()?;
+        let result = self.update_settings_txn(&mut txn, settings, update_builder)?;
+        txn.commit()?;
+        Ok(result)
+    }
+
+    pub fn delete_documents(
+        &self,
+        document_ids: &[String],
+        update_builder: UpdateBuilder,
+    ) -> Result<UpdateResult> {
+        let mut txn = self.write_txn()?;
+        let mut builder = update_builder.delete_documents(&mut txn, self)?;
+
+        // We ignore unexisting document ids
+        document_ids.iter().for_each(|id| {
+            builder.delete_external_id(id);
+        });
+
+        let deleted = builder.execute()?;
+        txn.commit()
+            .and(Ok(UpdateResult::DocumentDeletion { deleted }))
+            .map_err(Into::into)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_setting_check() {
+        // test no changes
+        let settings = Settings {
+            displayed_attributes: Some(Some(vec![String::from("hello")])),
+            searchable_attributes: Some(Some(vec![String::from("hello")])),
+            filterable_attributes: None,
+            ranking_rules: None,
+            stop_words: None,
+            synonyms: None,
+            distinct_attribute: None,
+            _kind: PhantomData::<Unchecked>,
+        };
+
+        let checked = settings.clone().check();
+        assert_eq!(settings.displayed_attributes, checked.displayed_attributes);
+        assert_eq!(
+            settings.searchable_attributes,
+            checked.searchable_attributes
+        );
+
+        // test wildcard
+        // test no changes
+        let settings = Settings {
+            displayed_attributes: Some(Some(vec![String::from("*")])),
+            searchable_attributes: Some(Some(vec![String::from("hello"), String::from("*")])),
+            filterable_attributes: None,
+            ranking_rules: None,
+            stop_words: None,
+            synonyms: None,
+            distinct_attribute: None,
+            _kind: PhantomData::<Unchecked>,
+        };
+
+        let checked = settings.check();
+        assert_eq!(checked.displayed_attributes, Some(None));
+        assert_eq!(checked.searchable_attributes, Some(None));
+    }
+}

meilisearch-http/src/index_controller/dump_actor/actor.rs

@@ -0,0 +1,157 @@
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use async_stream::stream;
+use chrono::Utc;
+use futures::{lock::Mutex, stream::StreamExt};
+use log::{error, trace};
+use tokio::sync::{mpsc, oneshot, RwLock};
+use update_actor::UpdateActorHandle;
+use uuid_resolver::UuidResolverHandle;
+
+use super::error::{DumpActorError, Result};
+use super::{DumpInfo, DumpMsg, DumpStatus, DumpTask};
+use crate::index_controller::{update_actor, uuid_resolver};
+
+pub const CONCURRENT_DUMP_MSG: usize = 10;
+
+pub struct DumpActor<UuidResolver, Update> {
+    inbox: Option<mpsc::Receiver<DumpMsg>>,
+    uuid_resolver: UuidResolver,
+    update: Update,
+    dump_path: PathBuf,
+    lock: Arc<Mutex<()>>,
+    dump_infos: Arc<RwLock<HashMap<String, DumpInfo>>>,
+    update_db_size: usize,
+    index_db_size: usize,
+}
+
+/// Generate uid from creation date
+fn generate_uid() -> String {
+    Utc::now().format("%Y%m%d-%H%M%S%3f").to_string()
+}
+
+impl<UuidResolver, Update> DumpActor<UuidResolver, Update>
+where
+    UuidResolver: UuidResolverHandle + Send + Sync + Clone + 'static,
+    Update: UpdateActorHandle + Send + Sync + Clone + 'static,
+{
+    pub fn new(
+        inbox: mpsc::Receiver<DumpMsg>,
+        uuid_resolver: UuidResolver,
+        update: Update,
+        dump_path: impl AsRef<Path>,
+        index_db_size: usize,
+        update_db_size: usize,
+    ) -> Self {
+        let dump_infos = Arc::new(RwLock::new(HashMap::new()));
+        let lock = Arc::new(Mutex::new(()));
+        Self {
+            inbox: Some(inbox),
+            uuid_resolver,
+            update,
+            dump_path: dump_path.as_ref().into(),
+            dump_infos,
+            lock,
+            index_db_size,
+            update_db_size,
+        }
+    }
+
+    pub async fn run(mut self) {
+        trace!("Started dump actor.");
+
+        let mut inbox = self
+            .inbox
+            .take()
+            .expect("Dump Actor must have a inbox at this point.");
+
+        let stream = stream! {
+            loop {
+                match inbox.recv().await {
+                    Some(msg) => yield msg,
+                    None => break,
+                }
+            }
+        };
+
+        stream
+            .for_each_concurrent(Some(CONCURRENT_DUMP_MSG), |msg| self.handle_message(msg))
+            .await;
+
+        error!("Dump actor stopped.");
+    }
+
+    async fn handle_message(&self, msg: DumpMsg) {
+        use DumpMsg::*;
+
+        match msg {
+            CreateDump { ret } => {
+                let _ = self.handle_create_dump(ret).await;
+            }
+            DumpInfo { ret, uid } => {
+                let _ = ret.send(self.handle_dump_info(uid).await);
+            }
+        }
+    }
+
+    async fn handle_create_dump(&self, ret: oneshot::Sender<Result<DumpInfo>>) {
+        let uid = generate_uid();
+        let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress);
+
+        let _lock = match self.lock.try_lock() {
+            Some(lock) => lock,
+            None => {
+                ret.send(Err(DumpActorError::DumpAlreadyRunning))
+                    .expect("Dump actor is dead");
+                return;
+            }
+        };
+
+        self.dump_infos
+            .write()
+            .await
+            .insert(uid.clone(), info.clone());
+
+        ret.send(Ok(info)).expect("Dump actor is dead");
+
+        let task = DumpTask {
+            path: self.dump_path.clone(),
+            uuid_resolver: self.uuid_resolver.clone(),
+            update_handle: self.update.clone(),
+            uid: uid.clone(),
+            update_db_size: self.update_db_size,
+            index_db_size: self.index_db_size,
+        };
+
+        let task_result = tokio::task::spawn(task.run()).await;
+
+        let mut dump_infos = self.dump_infos.write().await;
+        let dump_infos = dump_infos
+            .get_mut(&uid)
+            .expect("dump entry deleted while lock was acquired");
+
+        match task_result {
+            Ok(Ok(())) => {
+                dump_infos.done();
+                trace!("Dump succeed");
+            }
+            Ok(Err(e)) => {
+                dump_infos.with_error(e.to_string());
+                error!("Dump failed: {}", e);
+            }
+            Err(_) => {
+                dump_infos.with_error("Unexpected error while performing dump.".to_string());
+                error!("Dump panicked. Dump status set to failed");
+            }
+        };
+    }
+
+    async fn handle_dump_info(&self, uid: String) -> Result<DumpInfo> {
+        match self.dump_infos.read().await.get(&uid) {
+            Some(info) => Ok(info.clone()),
+            _ => Err(DumpActorError::DumpDoesNotExist(uid)),
+        }
+    }
+}

meilisearch-http/src/index_controller/dump_actor/error.rs

@@ -0,0 +1,52 @@
+use meilisearch_error::{Code, ErrorCode};
+
+use crate::index_controller::update_actor::error::UpdateActorError;
+use crate::index_controller::uuid_resolver::error::UuidResolverError;
+
+pub type Result<T> = std::result::Result<T, DumpActorError>;
+
+#[derive(thiserror::Error, Debug)]
+pub enum DumpActorError {
+    #[error("Another dump is already in progress")]
+    DumpAlreadyRunning,
+    #[error("Dump `{0}` not found")]
+    DumpDoesNotExist(String),
+    #[error("Internal error: {0}")]
+    Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
+    #[error("{0}")]
+    UuidResolver(#[from] UuidResolverError),
+    #[error("{0}")]
+    UpdateActor(#[from] UpdateActorError),
+}
+
+macro_rules! internal_error {
+    ($($other:path), *) => {
+        $(
+            impl From<$other> for DumpActorError {
+                fn from(other: $other) -> Self {
+                    Self::Internal(Box::new(other))
+                }
+            }
+        )*
+    }
+}
+
+internal_error!(
+    heed::Error,
+    std::io::Error,
+    tokio::task::JoinError,
+    serde_json::error::Error,
+    tempfile::PersistError
+);
+
+impl ErrorCode for DumpActorError {
+    fn error_code(&self) -> Code {
+        match self {
+            DumpActorError::DumpAlreadyRunning => Code::DumpAlreadyInProgress,
+            DumpActorError::DumpDoesNotExist(_) => Code::NotFound,
+            DumpActorError::Internal(_) => Code::Internal,
+            DumpActorError::UuidResolver(e) => e.error_code(),
+            DumpActorError::UpdateActor(e) => e.error_code(),
+        }
+    }
+}

meilisearch-http/src/index_controller/dump_actor/handle_impl.rs

@@ -0,0 +1,53 @@
+use std::path::Path;
+
+use actix_web::web::Bytes;
+use tokio::sync::{mpsc, oneshot};
+
+use super::error::Result;
+use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg};
+
+#[derive(Clone)]
+pub struct DumpActorHandleImpl {
+    sender: mpsc::Sender<DumpMsg>,
+}
+
+#[async_trait::async_trait]
+impl DumpActorHandle for DumpActorHandleImpl {
+    async fn create_dump(&self) -> Result<DumpInfo> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = DumpMsg::CreateDump { ret };
+        let _ = self.sender.send(msg).await;
+        receiver.await.expect("IndexActor has been killed")
+    }
+
+    async fn dump_info(&self, uid: String) -> Result<DumpInfo> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = DumpMsg::DumpInfo { ret, uid };
+        let _ = self.sender.send(msg).await;
+        receiver.await.expect("IndexActor has been killed")
+    }
+}
+
+impl DumpActorHandleImpl {
+    pub fn new(
+        path: impl AsRef<Path>,
+        uuid_resolver: crate::index_controller::uuid_resolver::UuidResolverHandleImpl,
+        update: crate::index_controller::update_actor::UpdateActorHandleImpl<Bytes>,
+        index_db_size: usize,
+        update_db_size: usize,
+    ) -> anyhow::Result<Self> {
+        let (sender, receiver) = mpsc::channel(10);
+        let actor = DumpActor::new(
+            receiver,
+            uuid_resolver,
+            update,
+            path,
+            index_db_size,
+            update_db_size,
+        );
+
+        tokio::task::spawn(actor.run());
+
+        Ok(Self { sender })
+    }
+}

meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs

@@ -0,0 +1,2 @@
+pub mod v1;
+pub mod v2;

meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs

@@ -0,0 +1,182 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs::{create_dir_all, File};
+use std::io::BufRead;
+use std::marker::PhantomData;
+use std::path::Path;
+use std::sync::Arc;
+
+use heed::EnvOpenOptions;
+use log::{error, info, warn};
+use milli::update::{IndexDocumentsMethod, UpdateFormat};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use crate::index_controller::{self, uuid_resolver::HeedUuidStore, IndexMetadata};
+use crate::{
+    index::{deserialize_some, update_handler::UpdateHandler, Index, Unchecked},
+    option::IndexerOpts,
+};
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct MetadataV1 {
+    db_version: String,
+    indexes: Vec<IndexMetadata>,
+}
+
+impl MetadataV1 {
+    pub fn load_dump(
+        self,
+        src: impl AsRef<Path>,
+        dst: impl AsRef<Path>,
+        size: usize,
+        indexer_options: &IndexerOpts,
+    ) -> anyhow::Result<()> {
+        info!(
+            "Loading dump, dump database version: {}, dump version: V1",
+            self.db_version
+        );
+
+        let uuid_store = HeedUuidStore::new(&dst)?;
+        for index in self.indexes {
+            let uuid = Uuid::new_v4();
+            uuid_store.insert(index.uid.clone(), uuid)?;
+            let src = src.as_ref().join(index.uid);
+            load_index(
+                &src,
+                &dst,
+                uuid,
+                index.meta.primary_key.as_deref(),
+                size,
+                indexer_options,
+            )?;
+        }
+
+        Ok(())
+    }
+}
+
+// These are the settings used in legacy meilisearch (<v0.21.0).
+#[derive(Default, Clone, Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct Settings {
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub ranking_rules: Option<Option<Vec<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub distinct_attribute: Option<Option<String>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub searchable_attributes: Option<Option<Vec<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub displayed_attributes: Option<Option<BTreeSet<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub stop_words: Option<Option<BTreeSet<String>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
+    #[serde(default, deserialize_with = "deserialize_some")]
+    pub filterable_attributes: Option<Option<Vec<String>>>,
+}
+
+fn load_index(
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    uuid: Uuid,
+    primary_key: Option<&str>,
+    size: usize,
+    indexer_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    let index_path = dst.as_ref().join(&format!("indexes/index-{}", uuid));
+
+    create_dir_all(&index_path)?;
+    let mut options = EnvOpenOptions::new();
+    options.map_size(size);
+    let index = milli::Index::new(options, index_path)?;
+    let index = Index(Arc::new(index));
+
+    // extract `settings.json` file and import content
+    let settings = import_settings(&src)?;
+    let settings: index_controller::Settings<Unchecked> = settings.into();
+
+    let mut txn = index.write_txn()?;
+
+    let handler = UpdateHandler::new(&indexer_options)?;
+
+    index.update_settings_txn(&mut txn, &settings.check(), handler.update_builder(0))?;
+
+    let file = File::open(&src.as_ref().join("documents.jsonl"))?;
+    let mut reader = std::io::BufReader::new(file);
+    reader.fill_buf()?;
+    if !reader.buffer().is_empty() {
+        index.update_documents_txn(
+            &mut txn,
+            UpdateFormat::JsonStream,
+            IndexDocumentsMethod::ReplaceDocuments,
+            Some(reader),
+            handler.update_builder(0),
+            primary_key,
+        )?;
+    }
+
+    txn.commit()?;
+
+    // Finaly, we extract the original milli::Index and close it
+    Arc::try_unwrap(index.0)
+        .map_err(|_e| "Couldn't close the index properly")
+        .unwrap()
+        .prepare_for_closing()
+        .wait();
+
+    // Updates are ignored in dumps V1.
+
+    Ok(())
+}
+
+/// we need to **always** be able to convert the old settings to the settings currently being used
+impl From<Settings> for index_controller::Settings<Unchecked> {
+    fn from(settings: Settings) -> Self {
+        Self {
+            distinct_attribute: settings.distinct_attribute,
+            // we need to convert the old `Vec<String>` into a `BTreeSet<String>`
+            displayed_attributes: settings.displayed_attributes.map(|o| o.map(|vec| vec.into_iter().collect())),
+            searchable_attributes: settings.searchable_attributes,
+            // we previously had a `Vec<String>` but now we have a `HashMap<String, String>`
+            // representing the name of the faceted field + the type of the field. Since the type
+            // was not known in the V1 of the dump we are just going to assume everything is a
+            // String
+            filterable_attributes: settings.filterable_attributes.map(|o| o.map(|vec| vec.into_iter().collect())),
+            // we need to convert the old `Vec<String>` into a `BTreeSet<String>`
+            ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| {
+                match criterion.as_str() {
+                    "words" | "typo" | "proximity" | "attribute" => Some(criterion),
+                    s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion),
+                    "wordsPosition" => {
+                        warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored");
+                        Some(String::from("words"))
+                    }
+                    "exactness" => {
+                        error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion);
+                        None
+                    }
+                    s => {
+                        error!("Unknown criterion found in the dump: `{}`, it will be ignored", s);
+                        None
+                    }
+                    }
+                }).collect())),
+            // we need to convert the old `Vec<String>` into a `BTreeSet<String>`
+            stop_words: settings.stop_words.map(|o| o.map(|vec| vec.into_iter().collect())),
+            // we need to convert the old `Vec<String>` into a `BTreeMap<String>`
+            synonyms: settings.synonyms.map(|o| o.map(|vec| vec.into_iter().collect())),
+            _kind: PhantomData,
+        }
+    }
+}
+
+/// Extract Settings from `settings.json` file present at provided `dir_path`
+fn import_settings(dir_path: impl AsRef<Path>) -> anyhow::Result<Settings> {
+    let path = dir_path.as_ref().join("settings.json");
+    let file = File::open(path)?;
+    let reader = std::io::BufReader::new(file);
+    let metadata = serde_json::from_reader(reader)?;
+
+    Ok(metadata)
+}

meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs

@@ -0,0 +1,59 @@
+use std::path::Path;
+
+use chrono::{DateTime, Utc};
+use log::info;
+use serde::{Deserialize, Serialize};
+
+use crate::index::Index;
+use crate::index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore};
+use crate::option::IndexerOpts;
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct MetadataV2 {
+    db_version: String,
+    index_db_size: usize,
+    update_db_size: usize,
+    dump_date: DateTime<Utc>,
+}
+
+impl MetadataV2 {
+    pub fn new(index_db_size: usize, update_db_size: usize) -> Self {
+        Self {
+            db_version: env!("CARGO_PKG_VERSION").to_string(),
+            index_db_size,
+            update_db_size,
+            dump_date: Utc::now(),
+        }
+    }
+
+    pub fn load_dump(
+        self,
+        src: impl AsRef<Path>,
+        dst: impl AsRef<Path>,
+        index_db_size: usize,
+        update_db_size: usize,
+        indexing_options: &IndexerOpts,
+    ) -> anyhow::Result<()> {
+        info!(
+            "Loading dump from {}, dump database version: {}, dump version: V2",
+            self.dump_date, self.db_version
+        );
+
+        info!("Loading index database.");
+        HeedUuidStore::load_dump(src.as_ref(), &dst)?;
+
+        info!("Loading updates.");
+        UpdateStore::load_dump(&src, &dst, update_db_size)?;
+
+        info!("Loading indexes.");
+        let indexes_path = src.as_ref().join("indexes");
+        let indexes = indexes_path.read_dir()?;
+        for index in indexes {
+            let index = index?;
+            Index::load_dump(&index.path(), &dst, index_db_size, indexing_options)?;
+        }
+
+        Ok(())
+    }
+}

meilisearch-http/src/index_controller/dump_actor/message.rs

@@ -0,0 +1,14 @@
+use tokio::sync::oneshot;
+
+use super::error::Result;
+use super::DumpInfo;
+
+pub enum DumpMsg {
+    CreateDump {
+        ret: oneshot::Sender<Result<DumpInfo>>,
+    },
+    DumpInfo {
+        uid: String,
+        ret: oneshot::Sender<Result<DumpInfo>>,
+    },
+}

meilisearch-http/src/index_controller/dump_actor/mod.rs

@@ -0,0 +1,203 @@
+use std::fs::File;
+use std::path::{Path, PathBuf};
+
+use anyhow::Context;
+use chrono::{DateTime, Utc};
+use log::{info, trace, warn};
+#[cfg(test)]
+use mockall::automock;
+use serde::{Deserialize, Serialize};
+use tokio::fs::create_dir_all;
+
+use loaders::v1::MetadataV1;
+use loaders::v2::MetadataV2;
+
+pub use actor::DumpActor;
+pub use handle_impl::*;
+pub use message::DumpMsg;
+
+use super::{update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle};
+use crate::index_controller::dump_actor::error::DumpActorError;
+use crate::{helpers::compression, option::IndexerOpts};
+use error::Result;
+
+mod actor;
+pub mod error;
+mod handle_impl;
+mod loaders;
+mod message;
+
+const META_FILE_NAME: &str = "metadata.json";
+
+#[async_trait::async_trait]
+#[cfg_attr(test, automock)]
+pub trait DumpActorHandle {
+    /// Start the creation of a dump
+    /// Implementation: [handle_impl::DumpActorHandleImpl::create_dump]
+    async fn create_dump(&self) -> Result<DumpInfo>;
+
+    /// Return the status of an already created dump
+    /// Implementation: [handle_impl::DumpActorHandleImpl::dump_status]
+    async fn dump_info(&self, uid: String) -> Result<DumpInfo>;
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "dumpVersion")]
+pub enum Metadata {
+    V1(MetadataV1),
+    V2(MetadataV2),
+}
+
+impl Metadata {
+    pub fn new_v2(index_db_size: usize, update_db_size: usize) -> Self {
+        let meta = MetadataV2::new(index_db_size, update_db_size);
+        Self::V2(meta)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum DumpStatus {
+    Done,
+    InProgress,
+    Failed,
+}
+
+#[derive(Debug, Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct DumpInfo {
+    pub uid: String,
+    pub status: DumpStatus,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+    started_at: DateTime<Utc>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    finished_at: Option<DateTime<Utc>>,
+}
+
+impl DumpInfo {
+    pub fn new(uid: String, status: DumpStatus) -> Self {
+        Self {
+            uid,
+            status,
+            error: None,
+            started_at: Utc::now(),
+            finished_at: None,
+        }
+    }
+
+    pub fn with_error(&mut self, error: String) {
+        self.status = DumpStatus::Failed;
+        self.finished_at = Some(Utc::now());
+        self.error = Some(error);
+    }
+
+    pub fn done(&mut self) {
+        self.finished_at = Some(Utc::now());
+        self.status = DumpStatus::Done;
+    }
+
+    pub fn dump_already_in_progress(&self) -> bool {
+        self.status == DumpStatus::InProgress
+    }
+}
+
+pub fn load_dump(
+    dst_path: impl AsRef<Path>,
+    src_path: impl AsRef<Path>,
+    index_db_size: usize,
+    update_db_size: usize,
+    indexer_opts: &IndexerOpts,
+) -> anyhow::Result<()> {
+    let tmp_src = tempfile::tempdir_in(".")?;
+    let tmp_src_path = tmp_src.path();
+
+    compression::from_tar_gz(&src_path, tmp_src_path)?;
+
+    let meta_path = tmp_src_path.join(META_FILE_NAME);
+    let mut meta_file = File::open(&meta_path)?;
+    let meta: Metadata = serde_json::from_reader(&mut meta_file)?;
+
+    let dst_dir = dst_path
+        .as_ref()
+        .parent()
+        .with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?;
+
+    let tmp_dst = tempfile::tempdir_in(dst_dir)?;
+
+    match meta {
+        Metadata::V1(meta) => {
+            meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size, indexer_opts)?
+        }
+        Metadata::V2(meta) => meta.load_dump(
+            &tmp_src_path,
+            tmp_dst.path(),
+            index_db_size,
+            update_db_size,
+            indexer_opts,
+        )?,
+    }
+    // Persist and atomically rename the db
+    let persisted_dump = tmp_dst.into_path();
+    if dst_path.as_ref().exists() {
+        warn!("Overwriting database at {}", dst_path.as_ref().display());
+        std::fs::remove_dir_all(&dst_path)?;
+    }
+
+    std::fs::rename(&persisted_dump, &dst_path)?;
+
+    Ok(())
+}
+
+struct DumpTask<U, P> {
+    path: PathBuf,
+    uuid_resolver: U,
+    update_handle: P,
+    uid: String,
+    update_db_size: usize,
+    index_db_size: usize,
+}
+
+impl<U, P> DumpTask<U, P>
+where
+    U: UuidResolverHandle + Send + Sync + Clone + 'static,
+    P: UpdateActorHandle + Send + Sync + Clone + 'static,
+{
+    async fn run(self) -> Result<()> {
+        trace!("Performing dump.");
+
+        create_dir_all(&self.path).await?;
+
+        let path_clone = self.path.clone();
+        let temp_dump_dir =
+            tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??;
+        let temp_dump_path = temp_dump_dir.path().to_owned();
+
+        let meta = Metadata::new_v2(self.index_db_size, self.update_db_size);
+        let meta_path = temp_dump_path.join(META_FILE_NAME);
+        let mut meta_file = File::create(&meta_path)?;
+        serde_json::to_writer(&mut meta_file, &meta)?;
+
+        let uuids = self.uuid_resolver.dump(temp_dump_path.clone()).await?;
+
+        self.update_handle
+            .dump(uuids, temp_dump_path.clone())
+            .await?;
+
+        let dump_path = tokio::task::spawn_blocking(move || -> Result<PathBuf> {
+            let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?;
+            compression::to_tar_gz(temp_dump_path, temp_dump_file.path())
+                .map_err(|e| DumpActorError::Internal(e.into()))?;
+
+            let dump_path = self.path.join(self.uid).with_extension("dump");
+            temp_dump_file.persist(&dump_path)?;
+
+            Ok(dump_path)
+        })
+        .await??;
+
+        info!("Created dump in {:?}.", dump_path);
+
+        Ok(())
+    }
+}

meilisearch-http/src/index_controller/error.rs

@@ -0,0 +1,40 @@
+use meilisearch_error::Code;
+use meilisearch_error::ErrorCode;
+
+use crate::index::error::IndexError;
+
+use super::dump_actor::error::DumpActorError;
+use super::index_actor::error::IndexActorError;
+use super::update_actor::error::UpdateActorError;
+use super::uuid_resolver::error::UuidResolverError;
+
+pub type Result<T> = std::result::Result<T, IndexControllerError>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum IndexControllerError {
+    #[error("Index creation must have an uid")]
+    MissingUid,
+    #[error("{0}")]
+    Uuid(#[from] UuidResolverError),
+    #[error("{0}")]
+    IndexActor(#[from] IndexActorError),
+    #[error("{0}")]
+    UpdateActor(#[from] UpdateActorError),
+    #[error("{0}")]
+    DumpActor(#[from] DumpActorError),
+    #[error("{0}")]
+    IndexError(#[from] IndexError),
+}
+
+impl ErrorCode for IndexControllerError {
+    fn error_code(&self) -> Code {
+        match self {
+            IndexControllerError::MissingUid => Code::BadRequest,
+            IndexControllerError::Uuid(e) => e.error_code(),
+            IndexControllerError::IndexActor(e) => e.error_code(),
+            IndexControllerError::UpdateActor(e) => e.error_code(),
+            IndexControllerError::DumpActor(e) => e.error_code(),
+            IndexControllerError::IndexError(e) => e.error_code(),
+        }
+    }
+}

meilisearch-http/src/index_controller/index_actor/actor.rs

@@ -0,0 +1,348 @@
+use std::fs::File;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use async_stream::stream;
+use futures::stream::StreamExt;
+use heed::CompactionOption;
+use log::debug;
+use milli::update::UpdateBuilder;
+use tokio::task::spawn_blocking;
+use tokio::{fs, sync::mpsc};
+use uuid::Uuid;
+
+use crate::index::{
+    update_handler::UpdateHandler, Checked, Document, SearchQuery, SearchResult, Settings,
+};
+use crate::index_controller::{
+    get_arc_ownership_blocking, Failed, IndexStats, Processed, Processing,
+};
+use crate::option::IndexerOpts;
+
+use super::error::{IndexActorError, Result};
+use super::{IndexMeta, IndexMsg, IndexSettings, IndexStore};
+
+pub const CONCURRENT_INDEX_MSG: usize = 10;
+
+pub struct IndexActor<S> {
+    receiver: Option<mpsc::Receiver<IndexMsg>>,
+    update_handler: Arc<UpdateHandler>,
+    store: S,
+}
+
+impl<S: IndexStore + Sync + Send> IndexActor<S> {
+    pub fn new(receiver: mpsc::Receiver<IndexMsg>, store: S) -> anyhow::Result<Self> {
+        let options = IndexerOpts::default();
+        let update_handler = UpdateHandler::new(&options)?;
+        let update_handler = Arc::new(update_handler);
+        let receiver = Some(receiver);
+        Ok(Self {
+            receiver,
+            update_handler,
+            store,
+        })
+    }
+
+    /// `run` poll the write_receiver and read_receiver concurrently, but while messages send
+    /// through the read channel are processed concurrently, the messages sent through the write
+    /// channel are processed one at a time.
+    pub async fn run(mut self) {
+        let mut receiver = self
+            .receiver
+            .take()
+            .expect("Index Actor must have a inbox at this point.");
+
+        let stream = stream! {
+            loop {
+                match receiver.recv().await {
+                    Some(msg) => yield msg,
+                    None => break,
+                }
+            }
+        };
+
+        stream
+            .for_each_concurrent(Some(CONCURRENT_INDEX_MSG), |msg| self.handle_message(msg))
+            .await;
+    }
+
+    async fn handle_message(&self, msg: IndexMsg) {
+        use IndexMsg::*;
+        match msg {
+            CreateIndex {
+                uuid,
+                primary_key,
+                ret,
+            } => {
+                let _ = ret.send(self.handle_create_index(uuid, primary_key).await);
+            }
+            Update {
+                ret,
+                meta,
+                data,
+                uuid,
+            } => {
+                let _ = ret.send(self.handle_update(uuid, meta, data).await);
+            }
+            Search { ret, query, uuid } => {
+                let _ = ret.send(self.handle_search(uuid, query).await);
+            }
+            Settings { ret, uuid } => {
+                let _ = ret.send(self.handle_settings(uuid).await);
+            }
+            Documents {
+                ret,
+                uuid,
+                attributes_to_retrieve,
+                offset,
+                limit,
+            } => {
+                let _ = ret.send(
+                    self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve)
+                        .await,
+                );
+            }
+            Document {
+                uuid,
+                attributes_to_retrieve,
+                doc_id,
+                ret,
+            } => {
+                let _ = ret.send(
+                    self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve)
+                        .await,
+                );
+            }
+            Delete { uuid, ret } => {
+                let _ = ret.send(self.handle_delete(uuid).await);
+            }
+            GetMeta { uuid, ret } => {
+                let _ = ret.send(self.handle_get_meta(uuid).await);
+            }
+            UpdateIndex {
+                uuid,
+                index_settings,
+                ret,
+            } => {
+                let _ = ret.send(self.handle_update_index(uuid, index_settings).await);
+            }
+            Snapshot { uuid, path, ret } => {
+                let _ = ret.send(self.handle_snapshot(uuid, path).await);
+            }
+            Dump { uuid, path, ret } => {
+                let _ = ret.send(self.handle_dump(uuid, path).await);
+            }
+            GetStats { uuid, ret } => {
+                let _ = ret.send(self.handle_get_stats(uuid).await);
+            }
+        }
+    }
+
+    async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult> {
+        let index = self
+            .store
+            .get(uuid)
+            .await?
+            .ok_or(IndexActorError::UnexistingIndex)?;
+        let result = spawn_blocking(move || index.perform_search(query)).await??;
+        Ok(result)
+    }
+
+    async fn handle_create_index(
+        &self,
+        uuid: Uuid,
+        primary_key: Option<String>,
+    ) -> Result<IndexMeta> {
+        let index = self.store.create(uuid, primary_key).await?;
+        let meta = spawn_blocking(move || IndexMeta::new(&index)).await??;
+        Ok(meta)
+    }
+
+    async fn handle_update(
+        &self,
+        uuid: Uuid,
+        meta: Processing,
+        data: Option<File>,
+    ) -> Result<std::result::Result<Processed, Failed>> {
+        debug!("Processing update {}", meta.id());
+        let update_handler = self.update_handler.clone();
+        let index = match self.store.get(uuid).await? {
+            Some(index) => index,
+            None => self.store.create(uuid, None).await?,
+        };
+
+        Ok(spawn_blocking(move || update_handler.handle_update(meta, data, index)).await?)
+    }
+
+    async fn handle_settings(&self, uuid: Uuid) -> Result<Settings<Checked>> {
+        let index = self
+            .store
+            .get(uuid)
+            .await?
+            .ok_or(IndexActorError::UnexistingIndex)?;
+        let result = spawn_blocking(move || index.settings()).await??;
+        Ok(result)
+    }
+
+    async fn handle_fetch_documents(
+        &self,
+        uuid: Uuid,
+        offset: usize,
+        limit: usize,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Vec<Document>> {
+        let index = self
+            .store
+            .get(uuid)
+            .await?
+            .ok_or(IndexActorError::UnexistingIndex)?;
+        let result =
+            spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve))
+                .await??;
+
+        Ok(result)
+    }
+
+    async fn handle_fetch_document(
+        &self,
+        uuid: Uuid,
+        doc_id: String,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Document> {
+        let index = self
+            .store
+            .get(uuid)
+            .await?
+            .ok_or(IndexActorError::UnexistingIndex)?;
+
+        let result =
+            spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve))
+                .await??;
+
+        Ok(result)
+    }
+
+    async fn handle_delete(&self, uuid: Uuid) -> Result<()> {
+        let index = self.store.delete(uuid).await?;
+
+        if let Some(index) = index {
+            tokio::task::spawn(async move {
+                let index = index.0;
+                let store = get_arc_ownership_blocking(index).await;
+                spawn_blocking(move || {
+                    store.prepare_for_closing().wait();
+                    debug!("Index closed");
+                });
+            });
+        }
+
+        Ok(())
+    }
+
+    async fn handle_get_meta(&self, uuid: Uuid) -> Result<IndexMeta> {
+        match self.store.get(uuid).await? {
+            Some(index) => {
+                let meta = spawn_blocking(move || IndexMeta::new(&index)).await??;
+                Ok(meta)
+            }
+            None => Err(IndexActorError::UnexistingIndex),
+        }
+    }
+
+    async fn handle_update_index(
+        &self,
+        uuid: Uuid,
+        index_settings: IndexSettings,
+    ) -> Result<IndexMeta> {
+        let index = self
+            .store
+            .get(uuid)
+            .await?
+            .ok_or(IndexActorError::UnexistingIndex)?;
+
+        let result = spawn_blocking(move || match index_settings.primary_key {
+            Some(primary_key) => {
+                let mut txn = index.write_txn()?;
+                if index.primary_key(&txn)?.is_some() {
+                    return Err(IndexActorError::ExistingPrimaryKey);
+                }
+                let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index);
+                builder.set_primary_key(primary_key);
+                builder.execute(|_, _| ())?;
+                let meta = IndexMeta::new_txn(&index, &txn)?;
+                txn.commit()?;
+                Ok(meta)
+            }
+            None => {
+                let meta = IndexMeta::new(&index)?;
+                Ok(meta)
+            }
+        })
+        .await??;
+
+        Ok(result)
+    }
+
+    async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> {
+        use tokio::fs::create_dir_all;
+
+        path.push("indexes");
+        create_dir_all(&path).await?;
+
+        if let Some(index) = self.store.get(uuid).await? {
+            let mut index_path = path.join(format!("index-{}", uuid));
+
+            create_dir_all(&index_path).await?;
+
+            index_path.push("data.mdb");
+            spawn_blocking(move || -> Result<()> {
+                // Get write txn to wait for ongoing write transaction before snapshot.
+                let _txn = index.write_txn()?;
+                index
+                    .env
+                    .copy_to_path(index_path, CompactionOption::Enabled)?;
+                Ok(())
+            })
+            .await??;
+        }
+
+        Ok(())
+    }
+
+    /// Create a `documents.jsonl` and a `settings.json` in `path/uid/` with a dump of all the
+    /// documents and all the settings.
+    async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
+        let index = self
+            .store
+            .get(uuid)
+            .await?
+            .ok_or(IndexActorError::UnexistingIndex)?;
+
+        let path = path.join(format!("indexes/index-{}/", uuid));
+        fs::create_dir_all(&path).await?;
+
+        tokio::task::spawn_blocking(move || index.dump(path)).await??;
+
+        Ok(())
+    }
+
+    async fn handle_get_stats(&self, uuid: Uuid) -> Result<IndexStats> {
+        let index = self
+            .store
+            .get(uuid)
+            .await?
+            .ok_or(IndexActorError::UnexistingIndex)?;
+
+        spawn_blocking(move || {
+            let rtxn = index.read_txn()?;
+
+            Ok(IndexStats {
+                size: index.size(),
+                number_of_documents: index.number_of_documents(&rtxn)?,
+                is_indexing: None,
+                field_distribution: index.field_distribution(&rtxn)?,
+            })
+        })
+        .await?
+    }
+}

meilisearch-http/src/index_controller/index_actor/error.rs

@@ -0,0 +1,48 @@
+use meilisearch_error::{Code, ErrorCode};
+
+use crate::{error::MilliError, index::error::IndexError};
+
+pub type Result<T> = std::result::Result<T, IndexActorError>;
+
+#[derive(thiserror::Error, Debug)]
+pub enum IndexActorError {
+    #[error("{0}")]
+    IndexError(#[from] IndexError),
+    #[error("Index already exists")]
+    IndexAlreadyExists,
+    #[error("Index not found")]
+    UnexistingIndex,
+    #[error("A primary key is already present. It's impossible to update it")]
+    ExistingPrimaryKey,
+    #[error("Internal Error: {0}")]
+    Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
+    #[error("{0}")]
+    Milli(#[from] milli::Error),
+}
+
+macro_rules! internal_error {
+    ($($other:path), *) => {
+        $(
+            impl From<$other> for IndexActorError {
+                fn from(other: $other) -> Self {
+                    Self::Internal(Box::new(other))
+                }
+            }
+        )*
+    }
+}
+
+internal_error!(heed::Error, tokio::task::JoinError, std::io::Error);
+
+impl ErrorCode for IndexActorError {
+    fn error_code(&self) -> Code {
+        match self {
+            IndexActorError::IndexError(e) => e.error_code(),
+            IndexActorError::IndexAlreadyExists => Code::IndexAlreadyExists,
+            IndexActorError::UnexistingIndex => Code::IndexNotFound,
+            IndexActorError::ExistingPrimaryKey => Code::PrimaryKeyAlreadyPresent,
+            IndexActorError::Internal(_) => Code::Internal,
+            IndexActorError::Milli(e) => MilliError(e).error_code(),
+        }
+    }
+}

meilisearch-http/src/index_controller/index_actor/handle_impl.rs

@@ -0,0 +1,159 @@
+use std::path::{Path, PathBuf};
+
+use tokio::sync::{mpsc, oneshot};
+use uuid::Uuid;
+
+use crate::{
+    index::Checked,
+    index_controller::{IndexSettings, IndexStats, Processing},
+};
+use crate::{
+    index::{Document, SearchQuery, SearchResult, Settings},
+    index_controller::{Failed, Processed},
+};
+
+use super::error::Result;
+use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore};
+
+#[derive(Clone)]
+pub struct IndexActorHandleImpl {
+    sender: mpsc::Sender<IndexMsg>,
+}
+
+#[async_trait::async_trait]
+impl IndexActorHandle for IndexActorHandleImpl {
+    async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::CreateIndex {
+            ret,
+            uuid,
+            primary_key,
+        };
+        let _ = self.sender.send(msg).await;
+        receiver.await.expect("IndexActor has been killed")
+    }
+
+    async fn update(
+        &self,
+        uuid: Uuid,
+        meta: Processing,
+        data: Option<std::fs::File>,
+    ) -> Result<std::result::Result<Processed, Failed>> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::Update {
+            ret,
+            meta,
+            data,
+            uuid,
+        };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::Search { uuid, query, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn settings(&self, uuid: Uuid) -> Result<Settings<Checked>> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::Settings { uuid, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn documents(
+        &self,
+        uuid: Uuid,
+        offset: usize,
+        limit: usize,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Vec<Document>> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::Documents {
+            uuid,
+            ret,
+            offset,
+            attributes_to_retrieve,
+            limit,
+        };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn document(
+        &self,
+        uuid: Uuid,
+        doc_id: String,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Document> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::Document {
+            uuid,
+            ret,
+            doc_id,
+            attributes_to_retrieve,
+        };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn delete(&self, uuid: Uuid) -> Result<()> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::Delete { uuid, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::GetMeta { uuid, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result<IndexMeta> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::UpdateIndex {
+            uuid,
+            index_settings,
+            ret,
+        };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::Snapshot { uuid, path, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::Dump { uuid, path, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+
+    async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = IndexMsg::GetStats { uuid, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver.await.expect("IndexActor has been killed")?)
+    }
+}
+
+impl IndexActorHandleImpl {
+    pub fn new(path: impl AsRef<Path>, index_size: usize) -> anyhow::Result<Self> {
+        let (sender, receiver) = mpsc::channel(100);
+
+        let store = MapIndexStore::new(path, index_size);
+        let actor = IndexActor::new(receiver, store)?;
+        tokio::task::spawn(actor.run());
+        Ok(Self { sender })
+    }
+}

meilisearch-http/src/index_controller/index_actor/message.rs

@@ -0,0 +1,74 @@
+use std::path::PathBuf;
+
+use tokio::sync::oneshot;
+use uuid::Uuid;
+
+use super::error::Result as IndexResult;
+use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
+use crate::index_controller::{Failed, IndexStats, Processed, Processing};
+
+use super::{IndexMeta, IndexSettings};
+
+#[allow(clippy::large_enum_variant)]
+pub enum IndexMsg {
+    CreateIndex {
+        uuid: Uuid,
+        primary_key: Option<String>,
+        ret: oneshot::Sender<IndexResult<IndexMeta>>,
+    },
+    Update {
+        uuid: Uuid,
+        meta: Processing,
+        data: Option<std::fs::File>,
+        ret: oneshot::Sender<IndexResult<Result<Processed, Failed>>>,
+    },
+    Search {
+        uuid: Uuid,
+        query: SearchQuery,
+        ret: oneshot::Sender<IndexResult<SearchResult>>,
+    },
+    Settings {
+        uuid: Uuid,
+        ret: oneshot::Sender<IndexResult<Settings<Checked>>>,
+    },
+    Documents {
+        uuid: Uuid,
+        attributes_to_retrieve: Option<Vec<String>>,
+        offset: usize,
+        limit: usize,
+        ret: oneshot::Sender<IndexResult<Vec<Document>>>,
+    },
+    Document {
+        uuid: Uuid,
+        attributes_to_retrieve: Option<Vec<String>>,
+        doc_id: String,
+        ret: oneshot::Sender<IndexResult<Document>>,
+    },
+    Delete {
+        uuid: Uuid,
+        ret: oneshot::Sender<IndexResult<()>>,
+    },
+    GetMeta {
+        uuid: Uuid,
+        ret: oneshot::Sender<IndexResult<IndexMeta>>,
+    },
+    UpdateIndex {
+        uuid: Uuid,
+        index_settings: IndexSettings,
+        ret: oneshot::Sender<IndexResult<IndexMeta>>,
+    },
+    Snapshot {
+        uuid: Uuid,
+        path: PathBuf,
+        ret: oneshot::Sender<IndexResult<()>>,
+    },
+    Dump {
+        uuid: Uuid,
+        path: PathBuf,
+        ret: oneshot::Sender<IndexResult<()>>,
+    },
+    GetStats {
+        uuid: Uuid,
+        ret: oneshot::Sender<IndexResult<IndexStats>>,
+    },
+}

meilisearch-http/src/index_controller/index_actor/mod.rs

@@ -0,0 +1,169 @@
+use std::fs::File;
+use std::path::PathBuf;
+
+use chrono::{DateTime, Utc};
+#[cfg(test)]
+use mockall::automock;
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use actor::IndexActor;
+pub use actor::CONCURRENT_INDEX_MSG;
+pub use handle_impl::IndexActorHandleImpl;
+use message::IndexMsg;
+use store::{IndexStore, MapIndexStore};
+
+use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings};
+use crate::index_controller::{Failed, IndexStats, Processed, Processing};
+use error::Result;
+
+use super::IndexSettings;
+
+mod actor;
+pub mod error;
+mod handle_impl;
+mod message;
+mod store;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct IndexMeta {
+    created_at: DateTime<Utc>,
+    pub updated_at: DateTime<Utc>,
+    pub primary_key: Option<String>,
+}
+
+impl IndexMeta {
+    fn new(index: &Index) -> Result<Self> {
+        let txn = index.read_txn()?;
+        Self::new_txn(index, &txn)
+    }
+
+    fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result<Self> {
+        let created_at = index.created_at(&txn)?;
+        let updated_at = index.updated_at(&txn)?;
+        let primary_key = index.primary_key(&txn)?.map(String::from);
+        Ok(Self {
+            created_at,
+            updated_at,
+            primary_key,
+        })
+    }
+}
+
+#[async_trait::async_trait]
+#[cfg_attr(test, automock)]
+pub trait IndexActorHandle {
+    async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta>;
+    async fn update(
+        &self,
+        uuid: Uuid,
+        meta: Processing,
+        data: Option<File>,
+    ) -> Result<std::result::Result<Processed, Failed>>;
+    async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult>;
+    async fn settings(&self, uuid: Uuid) -> Result<Settings<Checked>>;
+
+    async fn documents(
+        &self,
+        uuid: Uuid,
+        offset: usize,
+        limit: usize,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Vec<Document>>;
+    async fn document(
+        &self,
+        uuid: Uuid,
+        doc_id: String,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Document>;
+    async fn delete(&self, uuid: Uuid) -> Result<()>;
+    async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta>;
+    async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result<IndexMeta>;
+    async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
+    async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
+    async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats>;
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::Arc;
+
+    use super::*;
+
+    #[async_trait::async_trait]
+    /// Useful for passing around an `Arc<MockIndexActorHandle>` in tests.
+    impl IndexActorHandle for Arc<MockIndexActorHandle> {
+        async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> {
+            self.as_ref().create_index(uuid, primary_key).await
+        }
+
+        async fn update(
+            &self,
+            uuid: Uuid,
+            meta: Processing,
+            data: Option<std::fs::File>,
+        ) -> Result<std::result::Result<Processed, Failed>> {
+            self.as_ref().update(uuid, meta, data).await
+        }
+
+        async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult> {
+            self.as_ref().search(uuid, query).await
+        }
+
+        async fn settings(&self, uuid: Uuid) -> Result<Settings<Checked>> {
+            self.as_ref().settings(uuid).await
+        }
+
+        async fn documents(
+            &self,
+            uuid: Uuid,
+            offset: usize,
+            limit: usize,
+            attributes_to_retrieve: Option<Vec<String>>,
+        ) -> Result<Vec<Document>> {
+            self.as_ref()
+                .documents(uuid, offset, limit, attributes_to_retrieve)
+                .await
+        }
+
+        async fn document(
+            &self,
+            uuid: Uuid,
+            doc_id: String,
+            attributes_to_retrieve: Option<Vec<String>>,
+        ) -> Result<Document> {
+            self.as_ref()
+                .document(uuid, doc_id, attributes_to_retrieve)
+                .await
+        }
+
+        async fn delete(&self, uuid: Uuid) -> Result<()> {
+            self.as_ref().delete(uuid).await
+        }
+
+        async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta> {
+            self.as_ref().get_index_meta(uuid).await
+        }
+
+        async fn update_index(
+            &self,
+            uuid: Uuid,
+            index_settings: IndexSettings,
+        ) -> Result<IndexMeta> {
+            self.as_ref().update_index(uuid, index_settings).await
+        }
+
+        async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
+            self.as_ref().snapshot(uuid, path).await
+        }
+
+        async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
+            self.as_ref().dump(uuid, path).await
+        }
+
+        async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats> {
+            self.as_ref().get_index_stats(uuid).await
+        }
+    }
+}

meilisearch-http/src/index_controller/index_actor/store.rs

@@ -0,0 +1,103 @@
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use milli::update::UpdateBuilder;
+use tokio::fs;
+use tokio::sync::RwLock;
+use tokio::task::spawn_blocking;
+use uuid::Uuid;
+
+use super::error::{IndexActorError, Result};
+use crate::index::Index;
+
+type AsyncMap<K, V> = Arc<RwLock<HashMap<K, V>>>;
+
+#[async_trait::async_trait]
+pub trait IndexStore {
+    async fn create(&self, uuid: Uuid, primary_key: Option<String>) -> Result<Index>;
+    async fn get(&self, uuid: Uuid) -> Result<Option<Index>>;
+    async fn delete(&self, uuid: Uuid) -> Result<Option<Index>>;
+}
+
+pub struct MapIndexStore {
+    index_store: AsyncMap<Uuid, Index>,
+    path: PathBuf,
+    index_size: usize,
+}
+
+impl MapIndexStore {
+    pub fn new(path: impl AsRef<Path>, index_size: usize) -> Self {
+        let path = path.as_ref().join("indexes/");
+        let index_store = Arc::new(RwLock::new(HashMap::new()));
+        Self {
+            index_store,
+            path,
+            index_size,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl IndexStore for MapIndexStore {
+    async fn create(&self, uuid: Uuid, primary_key: Option<String>) -> Result<Index> {
+        // We need to keep the lock until we are sure the db file has been opened correclty, to
+        // ensure that another db is not created at the same time.
+        let mut lock = self.index_store.write().await;
+
+        if let Some(index) = lock.get(&uuid) {
+            return Ok(index.clone());
+        }
+        let path = self.path.join(format!("index-{}", uuid));
+        if path.exists() {
+            return Err(IndexActorError::IndexAlreadyExists);
+        }
+
+        let index_size = self.index_size;
+        let index = spawn_blocking(move || -> Result<Index> {
+            let index = Index::open(path, index_size)?;
+            if let Some(primary_key) = primary_key {
+                let mut txn = index.write_txn()?;
+
+                let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index);
+                builder.set_primary_key(primary_key);
+                builder.execute(|_, _| ())?;
+
+                txn.commit()?;
+            }
+            Ok(index)
+        })
+        .await??;
+
+        lock.insert(uuid, index.clone());
+
+        Ok(index)
+    }
+
+    async fn get(&self, uuid: Uuid) -> Result<Option<Index>> {
+        let guard = self.index_store.read().await;
+        match guard.get(&uuid) {
+            Some(index) => Ok(Some(index.clone())),
+            None => {
+                // drop the guard here so we can perform the write after without deadlocking;
+                drop(guard);
+                let path = self.path.join(format!("index-{}", uuid));
+                if !path.exists() {
+                    return Ok(None);
+                }
+
+                let index_size = self.index_size;
+                let index = spawn_blocking(move || Index::open(path, index_size)).await??;
+                self.index_store.write().await.insert(uuid, index.clone());
+                Ok(Some(index))
+            }
+        }
+    }
+
+    async fn delete(&self, uuid: Uuid) -> Result<Option<Index>> {
+        let db_path = self.path.join(format!("index-{}", uuid));
+        fs::remove_dir_all(db_path).await?;
+        let index = self.index_store.write().await.remove(&uuid);
+        Ok(index)
+    }
+}

meilisearch-http/src/index_controller/mod.rs

@@ -0,0 +1,441 @@
+use std::collections::BTreeMap;
+use std::path::Path;
+use std::sync::Arc;
+use std::time::Duration;
+
+use actix_web::web::Bytes;
+use chrono::{DateTime, Utc};
+use futures::stream::StreamExt;
+use log::error;
+use log::info;
+use milli::FieldDistribution;
+use serde::{Deserialize, Serialize};
+use tokio::sync::mpsc;
+use tokio::time::sleep;
+use uuid::Uuid;
+
+use dump_actor::DumpActorHandle;
+pub use dump_actor::{DumpInfo, DumpStatus};
+use index_actor::IndexActorHandle;
+use snapshot::{load_snapshot, SnapshotService};
+use update_actor::UpdateActorHandle;
+pub use updates::*;
+use uuid_resolver::{error::UuidResolverError, UuidResolverHandle};
+
+use crate::extractors::payload::Payload;
+use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
+use crate::option::Opt;
+use error::Result;
+
+use self::dump_actor::load_dump;
+use self::error::IndexControllerError;
+
+mod dump_actor;
+pub mod error;
+pub mod index_actor;
+mod snapshot;
+mod update_actor;
+mod updates;
+mod uuid_resolver;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct IndexMetadata {
+    #[serde(skip)]
+    pub uuid: Uuid,
+    pub uid: String,
+    name: String,
+    #[serde(flatten)]
+    pub meta: index_actor::IndexMeta,
+}
+
+#[derive(Clone, Debug)]
+pub struct IndexSettings {
+    pub uid: Option<String>,
+    pub primary_key: Option<String>,
+}
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct IndexStats {
+    #[serde(skip)]
+    pub size: u64,
+    pub number_of_documents: u64,
+    /// Whether the current index is performing an update. It is initially `None` when the
+    /// index returns it, since it is the `UpdateStore` that knows what index is currently indexing. It is
+    /// later set to either true or false, we we retrieve the information from the `UpdateStore`
+    pub is_indexing: Option<bool>,
+    pub field_distribution: FieldDistribution,
+}
+
+#[derive(Clone)]
+pub struct IndexController {
+    uuid_resolver: uuid_resolver::UuidResolverHandleImpl,
+    index_handle: index_actor::IndexActorHandleImpl,
+    update_handle: update_actor::UpdateActorHandleImpl<Bytes>,
+    dump_handle: dump_actor::DumpActorHandleImpl,
+}
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Stats {
+    pub database_size: u64,
+    pub last_update: Option<DateTime<Utc>>,
+    pub indexes: BTreeMap<String, IndexStats>,
+}
+
+impl IndexController {
+    pub fn new(path: impl AsRef<Path>, options: &Opt) -> anyhow::Result<Self> {
+        let index_size = options.max_index_size.get_bytes() as usize;
+        let update_store_size = options.max_index_size.get_bytes() as usize;
+
+        if let Some(ref path) = options.import_snapshot {
+            info!("Loading from snapshot {:?}", path);
+            load_snapshot(
+                &options.db_path,
+                path,
+                options.ignore_snapshot_if_db_exists,
+                options.ignore_missing_snapshot,
+            )?;
+        } else if let Some(ref src_path) = options.import_dump {
+            load_dump(
+                &options.db_path,
+                src_path,
+                options.max_index_size.get_bytes() as usize,
+                options.max_udb_size.get_bytes() as usize,
+                &options.indexer_options,
+            )?;
+        }
+
+        std::fs::create_dir_all(&path)?;
+
+        let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&path)?;
+        let index_handle = index_actor::IndexActorHandleImpl::new(&path, index_size)?;
+        let update_handle = update_actor::UpdateActorHandleImpl::new(
+            index_handle.clone(),
+            &path,
+            update_store_size,
+        )?;
+        let dump_handle = dump_actor::DumpActorHandleImpl::new(
+            &options.dumps_dir,
+            uuid_resolver.clone(),
+            update_handle.clone(),
+            options.max_index_size.get_bytes() as usize,
+            options.max_udb_size.get_bytes() as usize,
+        )?;
+
+        if options.schedule_snapshot {
+            let snapshot_service = SnapshotService::new(
+                uuid_resolver.clone(),
+                update_handle.clone(),
+                Duration::from_secs(options.snapshot_interval_sec),
+                options.snapshot_dir.clone(),
+                options
+                    .db_path
+                    .file_name()
+                    .map(|n| n.to_owned().into_string().expect("invalid path"))
+                    .unwrap_or_else(|| String::from("data.ms")),
+            );
+
+            tokio::task::spawn(snapshot_service.run());
+        }
+
+        Ok(Self {
+            uuid_resolver,
+            index_handle,
+            update_handle,
+            dump_handle,
+        })
+    }
+
+    pub async fn add_documents(
+        &self,
+        uid: String,
+        method: milli::update::IndexDocumentsMethod,
+        format: milli::update::UpdateFormat,
+        payload: Payload,
+        primary_key: Option<String>,
+    ) -> Result<UpdateStatus> {
+        let perform_update = |uuid| async move {
+            let meta = UpdateMeta::DocumentsAddition {
+                method,
+                format,
+                primary_key,
+            };
+            let (sender, receiver) = mpsc::channel(10);
+
+            // It is necessary to spawn a local task to send the payload to the update handle to
+            // prevent dead_locking between the update_handle::update that waits for the update to be
+            // registered and the update_actor that waits for the the payload to be sent to it.
+            tokio::task::spawn_local(async move {
+                payload
+                    .for_each(|r| async {
+                        let _ = sender.send(r).await;
+                    })
+                    .await
+            });
+
+            // This must be done *AFTER* spawning the task.
+            self.update_handle.update(meta, receiver, uuid).await
+        };
+
+        match self.uuid_resolver.get(uid).await {
+            Ok(uuid) => Ok(perform_update(uuid).await?),
+            Err(UuidResolverError::UnexistingIndex(name)) => {
+                let uuid = Uuid::new_v4();
+                let status = perform_update(uuid).await?;
+                // ignore if index creation fails now, since it may already have been created
+                let _ = self.index_handle.create_index(uuid, None).await;
+                self.uuid_resolver.insert(name, uuid).await?;
+                Ok(status)
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+
+    pub async fn clear_documents(&self, uid: String) -> Result<UpdateStatus> {
+        let uuid = self.uuid_resolver.get(uid).await?;
+        let meta = UpdateMeta::ClearDocuments;
+        let (_, receiver) = mpsc::channel(1);
+        let status = self.update_handle.update(meta, receiver, uuid).await?;
+        Ok(status)
+    }
+
+    pub async fn delete_documents(
+        &self,
+        uid: String,
+        documents: Vec<String>,
+    ) -> Result<UpdateStatus> {
+        let uuid = self.uuid_resolver.get(uid).await?;
+        let meta = UpdateMeta::DeleteDocuments { ids: documents };
+        let (_, receiver) = mpsc::channel(1);
+        let status = self.update_handle.update(meta, receiver, uuid).await?;
+        Ok(status)
+    }
+
+    pub async fn update_settings(
+        &self,
+        uid: String,
+        settings: Settings<Checked>,
+        create: bool,
+    ) -> Result<UpdateStatus> {
+        let perform_udpate = |uuid| async move {
+            let meta = UpdateMeta::Settings(settings.into_unchecked());
+            // Nothing so send, drop the sender right away, as not to block the update actor.
+            let (_, receiver) = mpsc::channel(1);
+            self.update_handle.update(meta, receiver, uuid).await
+        };
+
+        match self.uuid_resolver.get(uid).await {
+            Ok(uuid) => Ok(perform_udpate(uuid).await?),
+            Err(UuidResolverError::UnexistingIndex(name)) if create => {
+                let uuid = Uuid::new_v4();
+                let status = perform_udpate(uuid).await?;
+                // ignore if index creation fails now, since it may already have been created
+                let _ = self.index_handle.create_index(uuid, None).await;
+                self.uuid_resolver.insert(name, uuid).await?;
+                Ok(status)
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+
+    pub async fn create_index(&self, index_settings: IndexSettings) -> Result<IndexMetadata> {
+        let IndexSettings { uid, primary_key } = index_settings;
+        let uid = uid.ok_or(IndexControllerError::MissingUid)?;
+        let uuid = Uuid::new_v4();
+        let meta = self.index_handle.create_index(uuid, primary_key).await?;
+        self.uuid_resolver.insert(uid.clone(), uuid).await?;
+        let meta = IndexMetadata {
+            uuid,
+            name: uid.clone(),
+            uid,
+            meta,
+        };
+
+        Ok(meta)
+    }
+
+    pub async fn delete_index(&self, uid: String) -> Result<()> {
+        let uuid = self.uuid_resolver.delete(uid).await?;
+
+        // We remove the index from the resolver synchronously, and effectively perform the index
+        // deletion as a background task.
+        let update_handle = self.update_handle.clone();
+        let index_handle = self.index_handle.clone();
+        tokio::spawn(async move {
+            if let Err(e) = update_handle.delete(uuid).await {
+                error!("Error while deleting index: {}", e);
+            }
+            if let Err(e) = index_handle.delete(uuid).await {
+                error!("Error while deleting index: {}", e);
+            }
+        });
+
+        Ok(())
+    }
+
+    pub async fn update_status(&self, uid: String, id: u64) -> Result<UpdateStatus> {
+        let uuid = self.uuid_resolver.get(uid).await?;
+        let result = self.update_handle.update_status(uuid, id).await?;
+        Ok(result)
+    }
+
+    pub async fn all_update_status(&self, uid: String) -> Result<Vec<UpdateStatus>> {
+        let uuid = self.uuid_resolver.get(uid).await?;
+        let result = self.update_handle.get_all_updates_status(uuid).await?;
+        Ok(result)
+    }
+
+    pub async fn list_indexes(&self) -> Result<Vec<IndexMetadata>> {
+        let uuids = self.uuid_resolver.list().await?;
+
+        let mut ret = Vec::new();
+
+        for (uid, uuid) in uuids {
+            let meta = self.index_handle.get_index_meta(uuid).await?;
+            let meta = IndexMetadata {
+                uuid,
+                name: uid.clone(),
+                uid,
+                meta,
+            };
+            ret.push(meta);
+        }
+
+        Ok(ret)
+    }
+
+    pub async fn settings(&self, uid: String) -> Result<Settings<Checked>> {
+        let uuid = self.uuid_resolver.get(uid.clone()).await?;
+        let settings = self.index_handle.settings(uuid).await?;
+        Ok(settings)
+    }
+
+    pub async fn documents(
+        &self,
+        uid: String,
+        offset: usize,
+        limit: usize,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Vec<Document>> {
+        let uuid = self.uuid_resolver.get(uid.clone()).await?;
+        let documents = self
+            .index_handle
+            .documents(uuid, offset, limit, attributes_to_retrieve)
+            .await?;
+        Ok(documents)
+    }
+
+    pub async fn document(
+        &self,
+        uid: String,
+        doc_id: String,
+        attributes_to_retrieve: Option<Vec<String>>,
+    ) -> Result<Document> {
+        let uuid = self.uuid_resolver.get(uid.clone()).await?;
+        let document = self
+            .index_handle
+            .document(uuid, doc_id, attributes_to_retrieve)
+            .await?;
+        Ok(document)
+    }
+
+    pub async fn update_index(
+        &self,
+        uid: String,
+        mut index_settings: IndexSettings,
+    ) -> Result<IndexMetadata> {
+        if index_settings.uid.is_some() {
+            index_settings.uid.take();
+        }
+
+        let uuid = self.uuid_resolver.get(uid.clone()).await?;
+        let meta = self.index_handle.update_index(uuid, index_settings).await?;
+        let meta = IndexMetadata {
+            uuid,
+            name: uid.clone(),
+            uid,
+            meta,
+        };
+        Ok(meta)
+    }
+
+    pub async fn search(&self, uid: String, query: SearchQuery) -> Result<SearchResult> {
+        let uuid = self.uuid_resolver.get(uid).await?;
+        let result = self.index_handle.search(uuid, query).await?;
+        Ok(result)
+    }
+
+    pub async fn get_index(&self, uid: String) -> Result<IndexMetadata> {
+        let uuid = self.uuid_resolver.get(uid.clone()).await?;
+        let meta = self.index_handle.get_index_meta(uuid).await?;
+        let meta = IndexMetadata {
+            uuid,
+            name: uid.clone(),
+            uid,
+            meta,
+        };
+        Ok(meta)
+    }
+
+    pub async fn get_uuids_size(&self) -> Result<u64> {
+        Ok(self.uuid_resolver.get_size().await?)
+    }
+
+    pub async fn get_index_stats(&self, uid: String) -> Result<IndexStats> {
+        let uuid = self.uuid_resolver.get(uid).await?;
+        let update_infos = self.update_handle.get_info().await?;
+        let mut stats = self.index_handle.get_index_stats(uuid).await?;
+        // Check if the currently indexing update is from out index.
+        stats.is_indexing = Some(Some(uuid) == update_infos.processing);
+        Ok(stats)
+    }
+
+    pub async fn get_all_stats(&self) -> Result<Stats> {
+        let update_infos = self.update_handle.get_info().await?;
+        let mut database_size = self.get_uuids_size().await? + update_infos.size;
+        let mut last_update: Option<DateTime<_>> = None;
+        let mut indexes = BTreeMap::new();
+
+        for index in self.list_indexes().await? {
+            let mut index_stats = self.index_handle.get_index_stats(index.uuid).await?;
+            database_size += index_stats.size;
+
+            last_update = last_update.map_or(Some(index.meta.updated_at), |last| {
+                Some(last.max(index.meta.updated_at))
+            });
+
+            index_stats.is_indexing = Some(Some(index.uuid) == update_infos.processing);
+
+            indexes.insert(index.uid, index_stats);
+        }
+
+        Ok(Stats {
+            database_size,
+            last_update,
+            indexes,
+        })
+    }
+
+    pub async fn create_dump(&self) -> Result<DumpInfo> {
+        Ok(self.dump_handle.create_dump().await?)
+    }
+
+    pub async fn dump_info(&self, uid: String) -> Result<DumpInfo> {
+        Ok(self.dump_handle.dump_info(uid).await?)
+    }
+}
+
+pub async fn get_arc_ownership_blocking<T>(mut item: Arc<T>) -> T {
+    loop {
+        match Arc::try_unwrap(item) {
+            Ok(item) => return item,
+            Err(item_arc) => {
+                item = item_arc;
+                sleep(Duration::from_millis(100)).await;
+                continue;
+            }
+        }
+    }
+}

meilisearch-http/src/index_controller/snapshot.rs

@@ -0,0 +1,268 @@
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use anyhow::bail;
+use log::{error, info, trace};
+use tokio::fs;
+use tokio::task::spawn_blocking;
+use tokio::time::sleep;
+
+use super::update_actor::UpdateActorHandle;
+use super::uuid_resolver::UuidResolverHandle;
+use crate::helpers::compression;
+
+pub struct SnapshotService<U, R> {
+    uuid_resolver_handle: R,
+    update_handle: U,
+    snapshot_period: Duration,
+    snapshot_path: PathBuf,
+    db_name: String,
+}
+
+impl<U, R> SnapshotService<U, R>
+where
+    U: UpdateActorHandle,
+    R: UuidResolverHandle,
+{
+    pub fn new(
+        uuid_resolver_handle: R,
+        update_handle: U,
+        snapshot_period: Duration,
+        snapshot_path: PathBuf,
+        db_name: String,
+    ) -> Self {
+        Self {
+            uuid_resolver_handle,
+            update_handle,
+            snapshot_period,
+            snapshot_path,
+            db_name,
+        }
+    }
+
+    pub async fn run(self) {
+        info!(
+            "Snapshot scheduled every {}s.",
+            self.snapshot_period.as_secs()
+        );
+        loop {
+            if let Err(e) = self.perform_snapshot().await {
+                error!("Error while performing snapshot: {}", e);
+            }
+            sleep(self.snapshot_period).await;
+        }
+    }
+
+    async fn perform_snapshot(&self) -> anyhow::Result<()> {
+        trace!("Performing snapshot.");
+
+        let snapshot_dir = self.snapshot_path.clone();
+        fs::create_dir_all(&snapshot_dir).await?;
+        let temp_snapshot_dir =
+            spawn_blocking(move || tempfile::tempdir_in(snapshot_dir)).await??;
+        let temp_snapshot_path = temp_snapshot_dir.path().to_owned();
+
+        let uuids = self
+            .uuid_resolver_handle
+            .snapshot(temp_snapshot_path.clone())
+            .await?;
+
+        if uuids.is_empty() {
+            return Ok(());
+        }
+
+        self.update_handle
+            .snapshot(uuids, temp_snapshot_path.clone())
+            .await?;
+        let snapshot_dir = self.snapshot_path.clone();
+        let snapshot_path = self
+            .snapshot_path
+            .join(format!("{}.snapshot", self.db_name));
+        let snapshot_path = spawn_blocking(move || -> anyhow::Result<PathBuf> {
+            let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?;
+            let temp_snapshot_file_path = temp_snapshot_file.path().to_owned();
+            compression::to_tar_gz(temp_snapshot_path, temp_snapshot_file_path)?;
+            temp_snapshot_file.persist(&snapshot_path)?;
+            Ok(snapshot_path)
+        })
+        .await??;
+
+        trace!("Created snapshot in {:?}.", snapshot_path);
+
+        Ok(())
+    }
+}
+
+pub fn load_snapshot(
+    db_path: impl AsRef<Path>,
+    snapshot_path: impl AsRef<Path>,
+    ignore_snapshot_if_db_exists: bool,
+    ignore_missing_snapshot: bool,
+) -> anyhow::Result<()> {
+    if !db_path.as_ref().exists() && snapshot_path.as_ref().exists() {
+        match compression::from_tar_gz(snapshot_path, &db_path) {
+            Ok(()) => Ok(()),
+            Err(e) => {
+                // clean created db folder
+                std::fs::remove_dir_all(&db_path)?;
+                Err(e)
+            }
+        }
+    } else if db_path.as_ref().exists() && !ignore_snapshot_if_db_exists {
+        bail!(
+            "database already exists at {:?}, try to delete it or rename it",
+            db_path
+                .as_ref()
+                .canonicalize()
+                .unwrap_or_else(|_| db_path.as_ref().to_owned())
+        )
+    } else if !snapshot_path.as_ref().exists() && !ignore_missing_snapshot {
+        bail!(
+            "snapshot doesn't exist at {:?}",
+            snapshot_path
+                .as_ref()
+                .canonicalize()
+                .unwrap_or_else(|_| snapshot_path.as_ref().to_owned())
+        )
+    } else {
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::iter::FromIterator;
+    use std::{collections::HashSet, sync::Arc};
+
+    use futures::future::{err, ok};
+    use rand::Rng;
+    use tokio::time::timeout;
+    use uuid::Uuid;
+
+    use super::*;
+    use crate::index_controller::index_actor::MockIndexActorHandle;
+    use crate::index_controller::update_actor::{
+        error::UpdateActorError, MockUpdateActorHandle, UpdateActorHandleImpl,
+    };
+    use crate::index_controller::uuid_resolver::{
+        error::UuidResolverError, MockUuidResolverHandle,
+    };
+
+    #[actix_rt::test]
+    async fn test_normal() {
+        let mut rng = rand::thread_rng();
+        let uuids_num: usize = rng.gen_range(5, 10);
+        let uuids = (0..uuids_num)
+            .map(|_| Uuid::new_v4())
+            .collect::<HashSet<_>>();
+
+        let mut uuid_resolver = MockUuidResolverHandle::new();
+        let uuids_clone = uuids.clone();
+        uuid_resolver
+            .expect_snapshot()
+            .times(1)
+            .returning(move |_| Box::pin(ok(uuids_clone.clone())));
+
+        let uuids_clone = uuids.clone();
+        let mut index_handle = MockIndexActorHandle::new();
+        index_handle
+            .expect_snapshot()
+            .withf(move |uuid, _path| uuids_clone.contains(uuid))
+            .times(uuids_num)
+            .returning(move |_, _| Box::pin(ok(())));
+
+        let dir = tempfile::tempdir_in(".").unwrap();
+        let handle = Arc::new(index_handle);
+        let update_handle =
+            UpdateActorHandleImpl::<Vec<u8>>::new(handle.clone(), dir.path(), 4096 * 100).unwrap();
+
+        let snapshot_path = tempfile::tempdir_in(".").unwrap();
+        let snapshot_service = SnapshotService::new(
+            uuid_resolver,
+            update_handle,
+            Duration::from_millis(100),
+            snapshot_path.path().to_owned(),
+            "data.ms".to_string(),
+        );
+
+        snapshot_service.perform_snapshot().await.unwrap();
+    }
+
+    #[actix_rt::test]
+    async fn error_performing_uuid_snapshot() {
+        let mut uuid_resolver = MockUuidResolverHandle::new();
+        uuid_resolver
+            .expect_snapshot()
+            .times(1)
+            // abitrary error
+            .returning(|_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
+
+        let update_handle = MockUpdateActorHandle::new();
+
+        let snapshot_path = tempfile::tempdir_in(".").unwrap();
+        let snapshot_service = SnapshotService::new(
+            uuid_resolver,
+            update_handle,
+            Duration::from_millis(100),
+            snapshot_path.path().to_owned(),
+            "data.ms".to_string(),
+        );
+
+        assert!(snapshot_service.perform_snapshot().await.is_err());
+        // Nothing was written to the file
+        assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
+    }
+
+    #[actix_rt::test]
+    async fn error_performing_index_snapshot() {
+        let uuid = Uuid::new_v4();
+        let mut uuid_resolver = MockUuidResolverHandle::new();
+        uuid_resolver
+            .expect_snapshot()
+            .times(1)
+            .returning(move |_| Box::pin(ok(HashSet::from_iter(Some(uuid)))));
+
+        let mut update_handle = MockUpdateActorHandle::new();
+        update_handle
+            .expect_snapshot()
+            // abitrary error
+            .returning(|_, _| Box::pin(err(UpdateActorError::UnexistingUpdate(0))));
+
+        let snapshot_path = tempfile::tempdir_in(".").unwrap();
+        let snapshot_service = SnapshotService::new(
+            uuid_resolver,
+            update_handle,
+            Duration::from_millis(100),
+            snapshot_path.path().to_owned(),
+            "data.ms".to_string(),
+        );
+
+        assert!(snapshot_service.perform_snapshot().await.is_err());
+        // Nothing was written to the file
+        assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
+    }
+
+    #[actix_rt::test]
+    async fn test_loop() {
+        let mut uuid_resolver = MockUuidResolverHandle::new();
+        uuid_resolver
+            .expect_snapshot()
+            // we expect the funtion to be called between 2 and 3 time in the given interval.
+            .times(2..4)
+            // abitrary error, to short-circuit the function
+            .returning(move |_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
+
+        let update_handle = MockUpdateActorHandle::new();
+
+        let snapshot_path = tempfile::tempdir_in(".").unwrap();
+        let snapshot_service = SnapshotService::new(
+            uuid_resolver,
+            update_handle,
+            Duration::from_millis(100),
+            snapshot_path.path().to_owned(),
+            "data.ms".to_string(),
+        );
+
+        let _ = timeout(Duration::from_millis(300), snapshot_service.run()).await;
+    }
+}

meilisearch-http/src/index_controller/update_actor/actor.rs

@@ -0,0 +1,274 @@
+use std::collections::HashSet;
+use std::io::SeekFrom;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+
+use async_stream::stream;
+use futures::StreamExt;
+use log::trace;
+use oxidized_json_checker::JsonChecker;
+use tokio::fs;
+use tokio::io::AsyncWriteExt;
+use tokio::sync::mpsc;
+use uuid::Uuid;
+
+use super::error::{Result, UpdateActorError};
+use super::{PayloadData, UpdateMsg, UpdateStore, UpdateStoreInfo};
+use crate::index_controller::index_actor::IndexActorHandle;
+use crate::index_controller::{UpdateMeta, UpdateStatus};
+
+pub struct UpdateActor<D, I> {
+    path: PathBuf,
+    store: Arc<UpdateStore>,
+    inbox: Option<mpsc::Receiver<UpdateMsg<D>>>,
+    index_handle: I,
+    must_exit: Arc<AtomicBool>,
+}
+
+impl<D, I> UpdateActor<D, I>
+where
+    D: AsRef<[u8]> + Sized + 'static,
+    I: IndexActorHandle + Clone + Send + Sync + 'static,
+{
+    pub fn new(
+        update_db_size: usize,
+        inbox: mpsc::Receiver<UpdateMsg<D>>,
+        path: impl AsRef<Path>,
+        index_handle: I,
+    ) -> anyhow::Result<Self> {
+        let path = path.as_ref().join("updates");
+
+        std::fs::create_dir_all(&path)?;
+
+        let mut options = heed::EnvOpenOptions::new();
+        options.map_size(update_db_size);
+
+        let must_exit = Arc::new(AtomicBool::new(false));
+
+        let store = UpdateStore::open(options, &path, index_handle.clone(), must_exit.clone())?;
+        std::fs::create_dir_all(path.join("update_files"))?;
+        let inbox = Some(inbox);
+        Ok(Self {
+            path,
+            store,
+            inbox,
+            index_handle,
+            must_exit,
+        })
+    }
+
+    pub async fn run(mut self) {
+        use UpdateMsg::*;
+
+        trace!("Started update actor.");
+
+        let mut inbox = self
+            .inbox
+            .take()
+            .expect("A receiver should be present by now.");
+
+        let must_exit = self.must_exit.clone();
+        let stream = stream! {
+            loop {
+                let msg = inbox.recv().await;
+
+                if must_exit.load(std::sync::atomic::Ordering::Relaxed) {
+                    break;
+                }
+
+                match msg {
+                    Some(msg) => yield msg,
+                    None => break,
+                }
+            }
+        };
+
+        stream
+            .for_each_concurrent(Some(10), |msg| async {
+                match msg {
+                    Update {
+                        uuid,
+                        meta,
+                        data,
+                        ret,
+                    } => {
+                        let _ = ret.send(self.handle_update(uuid, meta, data).await);
+                    }
+                    ListUpdates { uuid, ret } => {
+                        let _ = ret.send(self.handle_list_updates(uuid).await);
+                    }
+                    GetUpdate { uuid, ret, id } => {
+                        let _ = ret.send(self.handle_get_update(uuid, id).await);
+                    }
+                    Delete { uuid, ret } => {
+                        let _ = ret.send(self.handle_delete(uuid).await);
+                    }
+                    Snapshot { uuids, path, ret } => {
+                        let _ = ret.send(self.handle_snapshot(uuids, path).await);
+                    }
+                    GetInfo { ret } => {
+                        let _ = ret.send(self.handle_get_info().await);
+                    }
+                    Dump { uuids, path, ret } => {
+                        let _ = ret.send(self.handle_dump(uuids, path).await);
+                    }
+                }
+            })
+            .await;
+    }
+
+    async fn handle_update(
+        &self,
+        uuid: Uuid,
+        meta: UpdateMeta,
+        payload: mpsc::Receiver<PayloadData<D>>,
+    ) -> Result<UpdateStatus> {
+        let file_path = match meta {
+            UpdateMeta::DocumentsAddition { .. } => {
+                let update_file_id = uuid::Uuid::new_v4();
+                let path = self
+                    .path
+                    .join(format!("update_files/update_{}", update_file_id));
+                let mut file = fs::OpenOptions::new()
+                    .read(true)
+                    .write(true)
+                    .create(true)
+                    .open(&path)
+                    .await?;
+
+                async fn write_to_file<D>(
+                    file: &mut fs::File,
+                    mut payload: mpsc::Receiver<PayloadData<D>>,
+                ) -> Result<usize>
+                where
+                    D: AsRef<[u8]> + Sized + 'static,
+                {
+                    let mut file_len = 0;
+
+                    while let Some(bytes) = payload.recv().await {
+                        let bytes = bytes?;
+                        file_len += bytes.as_ref().len();
+                        file.write_all(bytes.as_ref()).await?;
+                    }
+
+                    file.flush().await?;
+
+                    Ok(file_len)
+                }
+
+                let file_len = write_to_file(&mut file, payload).await;
+
+                match file_len {
+                    Ok(len) if len > 0 => {
+                        let file = file.into_std().await;
+                        Some((file, update_file_id))
+                    }
+                    Err(e) => {
+                        fs::remove_file(&path).await?;
+                        return Err(e);
+                    }
+                    _ => {
+                        fs::remove_file(&path).await?;
+                        None
+                    }
+                }
+            }
+            _ => None,
+        };
+
+        let update_store = self.store.clone();
+
+        tokio::task::spawn_blocking(move || {
+            use std::io::{copy, sink, BufReader, Seek};
+
+            // If the payload is empty, ignore the check.
+            let update_uuid = if let Some((mut file, uuid)) = file_path {
+                // set the file back to the beginning
+                file.seek(SeekFrom::Start(0))?;
+                // Check that the json payload is valid:
+                let reader = BufReader::new(&mut file);
+                let mut checker = JsonChecker::new(reader);
+
+                if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() {
+                    // The json file is invalid, we use Serde to get a nice error message:
+                    file.seek(SeekFrom::Start(0))?;
+                    let _: serde_json::Value = serde_json::from_reader(file)
+                        .map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?;
+                }
+                Some(uuid)
+            } else {
+                None
+            };
+
+            // The payload is valid, we can register it to the update store.
+            let status = update_store
+                .register_update(meta, update_uuid, uuid)
+                .map(UpdateStatus::Enqueued)?;
+            Ok(status)
+        })
+        .await?
+    }
+
+    async fn handle_list_updates(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
+        let update_store = self.store.clone();
+        tokio::task::spawn_blocking(move || {
+            let result = update_store.list(uuid)?;
+            Ok(result)
+        })
+        .await?
+    }
+
+    async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus> {
+        let store = self.store.clone();
+        tokio::task::spawn_blocking(move || {
+            let result = store
+                .meta(uuid, id)?
+                .ok_or(UpdateActorError::UnexistingUpdate(id))?;
+            Ok(result)
+        })
+        .await?
+    }
+
+    async fn handle_delete(&self, uuid: Uuid) -> Result<()> {
+        let store = self.store.clone();
+
+        tokio::task::spawn_blocking(move || store.delete_all(uuid)).await??;
+
+        Ok(())
+    }
+
+    async fn handle_snapshot(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
+        let index_handle = self.index_handle.clone();
+        let update_store = self.store.clone();
+
+        tokio::task::spawn_blocking(move || update_store.snapshot(&uuids, &path, index_handle))
+            .await??;
+
+        Ok(())
+    }
+
+    async fn handle_dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
+        let index_handle = self.index_handle.clone();
+        let update_store = self.store.clone();
+
+        tokio::task::spawn_blocking(move || -> Result<()> {
+            update_store.dump(&uuids, path.to_path_buf(), index_handle)?;
+            Ok(())
+        })
+        .await??;
+
+        Ok(())
+    }
+
+    async fn handle_get_info(&self) -> Result<UpdateStoreInfo> {
+        let update_store = self.store.clone();
+        let info = tokio::task::spawn_blocking(move || -> Result<UpdateStoreInfo> {
+            let info = update_store.get_info()?;
+            Ok(info)
+        })
+        .await??;
+
+        Ok(info)
+    }
+}

meilisearch-http/src/index_controller/update_actor/error.rs

@@ -0,0 +1,61 @@
+use std::error::Error;
+
+use meilisearch_error::{Code, ErrorCode};
+
+use crate::index_controller::index_actor::error::IndexActorError;
+
+pub type Result<T> = std::result::Result<T, UpdateActorError>;
+
+#[derive(Debug, thiserror::Error)]
+#[allow(clippy::large_enum_variant)]
+pub enum UpdateActorError {
+    #[error("Update {0} not found.")]
+    UnexistingUpdate(u64),
+    #[error("Internal error: {0}")]
+    Internal(Box<dyn Error + Send + Sync + 'static>),
+    #[error("{0}")]
+    IndexActor(#[from] IndexActorError),
+    #[error(
+        "update store was shut down due to a fatal error, please check your logs for more info."
+    )]
+    FatalUpdateStoreError,
+    #[error("{0}")]
+    InvalidPayload(Box<dyn Error + Send + Sync + 'static>),
+    #[error("{0}")]
+    PayloadError(#[from] actix_web::error::PayloadError),
+}
+
+impl<T> From<tokio::sync::mpsc::error::SendError<T>> for UpdateActorError {
+    fn from(_: tokio::sync::mpsc::error::SendError<T>) -> Self {
+        Self::FatalUpdateStoreError
+    }
+}
+
+impl From<tokio::sync::oneshot::error::RecvError> for UpdateActorError {
+    fn from(_: tokio::sync::oneshot::error::RecvError) -> Self {
+        Self::FatalUpdateStoreError
+    }
+}
+
+internal_error!(
+    UpdateActorError: heed::Error,
+    std::io::Error,
+    serde_json::Error,
+    tokio::task::JoinError
+);
+
+impl ErrorCode for UpdateActorError {
+    fn error_code(&self) -> Code {
+        match self {
+            UpdateActorError::UnexistingUpdate(_) => Code::NotFound,
+            UpdateActorError::Internal(_) => Code::Internal,
+            UpdateActorError::IndexActor(e) => e.error_code(),
+            UpdateActorError::FatalUpdateStoreError => Code::Internal,
+            UpdateActorError::InvalidPayload(_) => Code::BadRequest,
+            UpdateActorError::PayloadError(error) => match error {
+                actix_http::error::PayloadError::Overflow => Code::PayloadTooLarge,
+                _ => Code::Internal,
+            },
+        }
+    }
+}

meilisearch-http/src/index_controller/update_actor/handle_impl.rs

@@ -0,0 +1,103 @@
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+
+use tokio::sync::{mpsc, oneshot};
+use uuid::Uuid;
+
+use crate::index_controller::{IndexActorHandle, UpdateStatus};
+
+use super::error::Result;
+use super::{PayloadData, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStoreInfo};
+
+#[derive(Clone)]
+pub struct UpdateActorHandleImpl<D> {
+    sender: mpsc::Sender<UpdateMsg<D>>,
+}
+
+impl<D> UpdateActorHandleImpl<D>
+where
+    D: AsRef<[u8]> + Sized + 'static + Sync + Send,
+{
+    pub fn new<I>(
+        index_handle: I,
+        path: impl AsRef<Path>,
+        update_store_size: usize,
+    ) -> anyhow::Result<Self>
+    where
+        I: IndexActorHandle + Clone + Send + Sync + 'static,
+    {
+        let path = path.as_ref().to_owned();
+        let (sender, receiver) = mpsc::channel(100);
+        let actor = UpdateActor::new(update_store_size, receiver, path, index_handle)?;
+
+        tokio::task::spawn(actor.run());
+
+        Ok(Self { sender })
+    }
+}
+
+#[async_trait::async_trait]
+impl<D> UpdateActorHandle for UpdateActorHandleImpl<D>
+where
+    D: AsRef<[u8]> + Sized + 'static + Sync + Send,
+{
+    type Data = D;
+
+    async fn get_all_updates_status(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UpdateMsg::ListUpdates { uuid, ret };
+        self.sender.send(msg).await?;
+        receiver.await?
+    }
+    async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UpdateMsg::GetUpdate { uuid, id, ret };
+        self.sender.send(msg).await?;
+        receiver.await?
+    }
+
+    async fn delete(&self, uuid: Uuid) -> Result<()> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UpdateMsg::Delete { uuid, ret };
+        self.sender.send(msg).await?;
+        receiver.await?
+    }
+
+    async fn snapshot(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UpdateMsg::Snapshot { uuids, path, ret };
+        self.sender.send(msg).await?;
+        receiver.await?
+    }
+
+    async fn dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UpdateMsg::Dump { uuids, path, ret };
+        self.sender.send(msg).await?;
+        receiver.await?
+    }
+
+    async fn get_info(&self) -> Result<UpdateStoreInfo> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UpdateMsg::GetInfo { ret };
+        self.sender.send(msg).await?;
+        receiver.await?
+    }
+
+    async fn update(
+        &self,
+        meta: UpdateMeta,
+        data: mpsc::Receiver<PayloadData<Self::Data>>,
+        uuid: Uuid,
+    ) -> Result<UpdateStatus> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UpdateMsg::Update {
+            uuid,
+            data,
+            meta,
+            ret,
+        };
+        self.sender.send(msg).await?;
+        receiver.await?
+    }
+}

meilisearch-http/src/index_controller/update_actor/message.rs

@@ -0,0 +1,43 @@
+use std::collections::HashSet;
+use std::path::PathBuf;
+
+use tokio::sync::{mpsc, oneshot};
+use uuid::Uuid;
+
+use super::error::Result;
+use super::{PayloadData, UpdateMeta, UpdateStatus, UpdateStoreInfo};
+
+pub enum UpdateMsg<D> {
+    Update {
+        uuid: Uuid,
+        meta: UpdateMeta,
+        data: mpsc::Receiver<PayloadData<D>>,
+        ret: oneshot::Sender<Result<UpdateStatus>>,
+    },
+    ListUpdates {
+        uuid: Uuid,
+        ret: oneshot::Sender<Result<Vec<UpdateStatus>>>,
+    },
+    GetUpdate {
+        uuid: Uuid,
+        ret: oneshot::Sender<Result<UpdateStatus>>,
+        id: u64,
+    },
+    Delete {
+        uuid: Uuid,
+        ret: oneshot::Sender<Result<()>>,
+    },
+    Snapshot {
+        uuids: HashSet<Uuid>,
+        path: PathBuf,
+        ret: oneshot::Sender<Result<()>>,
+    },
+    Dump {
+        uuids: HashSet<Uuid>,
+        path: PathBuf,
+        ret: oneshot::Sender<Result<()>>,
+    },
+    GetInfo {
+        ret: oneshot::Sender<Result<UpdateStoreInfo>>,
+    },
+}

meilisearch-http/src/index_controller/update_actor/mod.rs

@@ -0,0 +1,44 @@
+use std::{collections::HashSet, path::PathBuf};
+
+use actix_http::error::PayloadError;
+use tokio::sync::mpsc;
+use uuid::Uuid;
+
+use crate::index_controller::{UpdateMeta, UpdateStatus};
+
+use actor::UpdateActor;
+use error::Result;
+use message::UpdateMsg;
+
+pub use handle_impl::UpdateActorHandleImpl;
+pub use store::{UpdateStore, UpdateStoreInfo};
+
+mod actor;
+pub mod error;
+mod handle_impl;
+mod message;
+pub mod store;
+
+type PayloadData<D> = std::result::Result<D, PayloadError>;
+
+#[cfg(test)]
+use mockall::automock;
+
+#[async_trait::async_trait]
+#[cfg_attr(test, automock(type Data=Vec<u8>;))]
+pub trait UpdateActorHandle {
+    type Data: AsRef<[u8]> + Sized + 'static + Sync + Send;
+
+    async fn get_all_updates_status(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>>;
+    async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus>;
+    async fn delete(&self, uuid: Uuid) -> Result<()>;
+    async fn snapshot(&self, uuid: HashSet<Uuid>, path: PathBuf) -> Result<()>;
+    async fn dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()>;
+    async fn get_info(&self) -> Result<UpdateStoreInfo>;
+    async fn update(
+        &self,
+        meta: UpdateMeta,
+        data: mpsc::Receiver<PayloadData<Self::Data>>,
+        uuid: Uuid,
+    ) -> Result<UpdateStatus>;
+}

meilisearch-http/src/index_controller/update_actor/store/codec.rs

@@ -0,0 +1,86 @@
+use std::{borrow::Cow, convert::TryInto, mem::size_of};
+
+use heed::{BytesDecode, BytesEncode};
+use uuid::Uuid;
+
+pub struct NextIdCodec;
+
+pub enum NextIdKey {
+    Global,
+    Index(Uuid),
+}
+
+impl<'a> BytesEncode<'a> for NextIdCodec {
+    type EItem = NextIdKey;
+
+    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
+        match item {
+            NextIdKey::Global => Some(Cow::Borrowed(b"__global__")),
+            NextIdKey::Index(ref uuid) => Some(Cow::Borrowed(uuid.as_bytes())),
+        }
+    }
+}
+
+pub struct PendingKeyCodec;
+
+impl<'a> BytesEncode<'a> for PendingKeyCodec {
+    type EItem = (u64, Uuid, u64);
+
+    fn bytes_encode((global_id, uuid, update_id): &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
+        let mut bytes = Vec::with_capacity(size_of::<Self::EItem>());
+        bytes.extend_from_slice(&global_id.to_be_bytes());
+        bytes.extend_from_slice(uuid.as_bytes());
+        bytes.extend_from_slice(&update_id.to_be_bytes());
+        Some(Cow::Owned(bytes))
+    }
+}
+
+impl<'a> BytesDecode<'a> for PendingKeyCodec {
+    type DItem = (u64, Uuid, u64);
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        let global_id_bytes = bytes.get(0..size_of::<u64>())?.try_into().ok()?;
+        let global_id = u64::from_be_bytes(global_id_bytes);
+
+        let uuid_bytes = bytes
+            .get(size_of::<u64>()..(size_of::<u64>() + size_of::<Uuid>()))?
+            .try_into()
+            .ok()?;
+        let uuid = Uuid::from_bytes(uuid_bytes);
+
+        let update_id_bytes = bytes
+            .get((size_of::<u64>() + size_of::<Uuid>())..)?
+            .try_into()
+            .ok()?;
+        let update_id = u64::from_be_bytes(update_id_bytes);
+
+        Some((global_id, uuid, update_id))
+    }
+}
+
+pub struct UpdateKeyCodec;
+
+impl<'a> BytesEncode<'a> for UpdateKeyCodec {
+    type EItem = (Uuid, u64);
+
+    fn bytes_encode((uuid, update_id): &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
+        let mut bytes = Vec::with_capacity(size_of::<Self::EItem>());
+        bytes.extend_from_slice(uuid.as_bytes());
+        bytes.extend_from_slice(&update_id.to_be_bytes());
+        Some(Cow::Owned(bytes))
+    }
+}
+
+impl<'a> BytesDecode<'a> for UpdateKeyCodec {
+    type DItem = (Uuid, u64);
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        let uuid_bytes = bytes.get(0..size_of::<Uuid>())?.try_into().ok()?;
+        let uuid = Uuid::from_bytes(uuid_bytes);
+
+        let update_id_bytes = bytes.get(size_of::<Uuid>()..)?.try_into().ok()?;
+        let update_id = u64::from_be_bytes(update_id_bytes);
+
+        Some((uuid, update_id))
+    }
+}

meilisearch-http/src/index_controller/update_actor/store/dump.rs

@@ -0,0 +1,184 @@
+use std::{
+    collections::HashSet,
+    fs::{create_dir_all, File},
+    io::{BufRead, BufReader, Write},
+    path::{Path, PathBuf},
+};
+
+use heed::{EnvOpenOptions, RoTxn};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use super::{Result, State, UpdateStore};
+use crate::index_controller::{
+    index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued,
+    UpdateStatus,
+};
+
+#[derive(Serialize, Deserialize)]
+struct UpdateEntry {
+    uuid: Uuid,
+    update: UpdateStatus,
+}
+
+impl UpdateStore {
+    pub fn dump(
+        &self,
+        uuids: &HashSet<Uuid>,
+        path: PathBuf,
+        handle: impl IndexActorHandle,
+    ) -> Result<()> {
+        let state_lock = self.state.write();
+        state_lock.swap(State::Dumping);
+
+        // txn must *always* be acquired after state lock, or it will dead lock.
+        let txn = self.env.write_txn()?;
+
+        let dump_path = path.join("updates");
+        create_dir_all(&dump_path)?;
+
+        self.dump_updates(&txn, uuids, &dump_path)?;
+
+        let fut = dump_indexes(uuids, handle, &path);
+        tokio::runtime::Handle::current().block_on(fut)?;
+
+        state_lock.swap(State::Idle);
+
+        Ok(())
+    }
+
+    fn dump_updates(
+        &self,
+        txn: &RoTxn,
+        uuids: &HashSet<Uuid>,
+        path: impl AsRef<Path>,
+    ) -> Result<()> {
+        let dump_data_path = path.as_ref().join("data.jsonl");
+        let mut dump_data_file = File::create(dump_data_path)?;
+
+        let update_files_path = path.as_ref().join(super::UPDATE_DIR);
+        create_dir_all(&update_files_path)?;
+
+        self.dump_pending(&txn, uuids, &mut dump_data_file, &path)?;
+        self.dump_completed(&txn, uuids, &mut dump_data_file)?;
+
+        Ok(())
+    }
+
+    fn dump_pending(
+        &self,
+        txn: &RoTxn,
+        uuids: &HashSet<Uuid>,
+        mut file: &mut File,
+        dst_path: impl AsRef<Path>,
+    ) -> Result<()> {
+        let pendings = self.pending_queue.iter(txn)?.lazily_decode_data();
+
+        for pending in pendings {
+            let ((_, uuid, _), data) = pending?;
+            if uuids.contains(&uuid) {
+                let update = data.decode()?;
+
+                if let Some(ref update_uuid) = update.content {
+                    let src = super::update_uuid_to_file_path(&self.path, *update_uuid);
+                    let dst = super::update_uuid_to_file_path(&dst_path, *update_uuid);
+                    std::fs::copy(src, dst)?;
+                }
+
+                let update_json = UpdateEntry {
+                    uuid,
+                    update: update.into(),
+                };
+
+                serde_json::to_writer(&mut file, &update_json)?;
+                file.write_all(b"\n")?;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn dump_completed(
+        &self,
+        txn: &RoTxn,
+        uuids: &HashSet<Uuid>,
+        mut file: &mut File,
+    ) -> Result<()> {
+        let updates = self.updates.iter(txn)?.lazily_decode_data();
+
+        for update in updates {
+            let ((uuid, _), data) = update?;
+            if uuids.contains(&uuid) {
+                let update = data.decode()?;
+
+                let update_json = UpdateEntry { uuid, update };
+
+                serde_json::to_writer(&mut file, &update_json)?;
+                file.write_all(b"\n")?;
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn load_dump(
+        src: impl AsRef<Path>,
+        dst: impl AsRef<Path>,
+        db_size: usize,
+    ) -> anyhow::Result<()> {
+        let dst_update_path = dst.as_ref().join("updates/");
+        create_dir_all(&dst_update_path)?;
+
+        let mut options = EnvOpenOptions::new();
+        options.map_size(db_size as usize);
+        let (store, _) = UpdateStore::new(options, &dst_update_path)?;
+
+        let src_update_path = src.as_ref().join("updates");
+        let update_data = File::open(&src_update_path.join("data.jsonl"))?;
+        let mut update_data = BufReader::new(update_data);
+
+        std::fs::create_dir_all(dst_update_path.join("update_files/"))?;
+
+        let mut wtxn = store.env.write_txn()?;
+        let mut line = String::new();
+        loop {
+            match update_data.read_line(&mut line) {
+                Ok(0) => break,
+                Ok(_) => {
+                    let UpdateEntry { uuid, update } = serde_json::from_str(&line)?;
+                    store.register_raw_updates(&mut wtxn, &update, uuid)?;
+
+                    // Copy ascociated update path if it exists
+                    if let UpdateStatus::Enqueued(Enqueued {
+                        content: Some(uuid),
+                        ..
+                    }) = update
+                    {
+                        let src = update_uuid_to_file_path(&src_update_path, uuid);
+                        let dst = update_uuid_to_file_path(&dst_update_path, uuid);
+                        std::fs::copy(src, dst)?;
+                    }
+                }
+                _ => break,
+            }
+
+            line.clear();
+        }
+
+        wtxn.commit()?;
+
+        Ok(())
+    }
+}
+
+async fn dump_indexes(
+    uuids: &HashSet<Uuid>,
+    handle: impl IndexActorHandle,
+    path: impl AsRef<Path>,
+) -> Result<()> {
+    for uuid in uuids {
+        handle.dump(*uuid, path.as_ref().to_owned()).await?;
+    }
+
+    Ok(())
+}

meilisearch-http/src/index_controller/update_actor/store/mod.rs

@@ -0,0 +1,721 @@
+mod codec;
+pub mod dump;
+
+use std::fs::{copy, create_dir_all, remove_file, File};
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::{
+    collections::{BTreeMap, HashSet},
+    path::PathBuf,
+};
+
+use arc_swap::ArcSwap;
+use futures::StreamExt;
+use heed::types::{ByteSlice, OwnedType, SerdeJson};
+use heed::zerocopy::U64;
+use heed::{CompactionOption, Database, Env, EnvOpenOptions};
+use log::error;
+use parking_lot::{Mutex, MutexGuard};
+use tokio::runtime::Handle;
+use tokio::sync::mpsc;
+use uuid::Uuid;
+
+use codec::*;
+
+use super::error::Result;
+use super::UpdateMeta;
+use crate::helpers::EnvSizer;
+use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle};
+
+#[allow(clippy::upper_case_acronyms)]
+type BEU64 = U64<heed::byteorder::BE>;
+
+const UPDATE_DIR: &str = "update_files";
+
+pub struct UpdateStoreInfo {
+    /// Size of the update store in bytes.
+    pub size: u64,
+    /// Uuid of the currently processing update if it exists
+    pub processing: Option<Uuid>,
+}
+
+/// A data structure that allows concurrent reads AND exactly one writer.
+pub struct StateLock {
+    lock: Mutex<()>,
+    data: ArcSwap<State>,
+}
+
+pub struct StateLockGuard<'a> {
+    _lock: MutexGuard<'a, ()>,
+    state: &'a StateLock,
+}
+
+impl StateLockGuard<'_> {
+    pub fn swap(&self, state: State) -> Arc<State> {
+        self.state.data.swap(Arc::new(state))
+    }
+}
+
+impl StateLock {
+    fn from_state(state: State) -> Self {
+        let lock = Mutex::new(());
+        let data = ArcSwap::from(Arc::new(state));
+        Self { lock, data }
+    }
+
+    pub fn read(&self) -> Arc<State> {
+        self.data.load().clone()
+    }
+
+    pub fn write(&self) -> StateLockGuard {
+        let _lock = self.lock.lock();
+        let state = &self;
+        StateLockGuard { _lock, state }
+    }
+}
+
+#[allow(clippy::large_enum_variant)]
+pub enum State {
+    Idle,
+    Processing(Uuid, Processing),
+    Snapshoting,
+    Dumping,
+}
+
+#[derive(Clone)]
+pub struct UpdateStore {
+    pub env: Env,
+    /// A queue containing the updates to process, ordered by arrival.
+    /// The key are built as follow:
+    /// | global_update_id | index_uuid | update_id |
+    /// |     8-bytes      |  16-bytes  |  8-bytes  |
+    pending_queue: Database<PendingKeyCodec, SerdeJson<Enqueued>>,
+    /// Map indexes to the next available update id. If NextIdKey::Global is queried, then the next
+    /// global update id is returned
+    next_update_id: Database<NextIdCodec, OwnedType<BEU64>>,
+    /// Contains all the performed updates meta, be they failed, aborted, or processed.
+    /// The keys are built as follow:
+    /// |    Uuid  |   id    |
+    /// | 16-bytes | 8-bytes |
+    updates: Database<UpdateKeyCodec, SerdeJson<UpdateStatus>>,
+    /// Indicates the current state of the update store,
+    state: Arc<StateLock>,
+    /// Wake up the loop when a new event occurs.
+    notification_sender: mpsc::Sender<()>,
+    path: PathBuf,
+}
+
+impl UpdateStore {
+    fn new(
+        mut options: EnvOpenOptions,
+        path: impl AsRef<Path>,
+    ) -> anyhow::Result<(Self, mpsc::Receiver<()>)> {
+        options.max_dbs(5);
+
+        let env = options.open(&path)?;
+        let pending_queue = env.create_database(Some("pending-queue"))?;
+        let next_update_id = env.create_database(Some("next-update-id"))?;
+        let updates = env.create_database(Some("updates"))?;
+
+        let state = Arc::new(StateLock::from_state(State::Idle));
+
+        let (notification_sender, notification_receiver) = mpsc::channel(10);
+
+        Ok((
+            Self {
+                env,
+                pending_queue,
+                next_update_id,
+                updates,
+                state,
+                notification_sender,
+                path: path.as_ref().to_owned(),
+            },
+            notification_receiver,
+        ))
+    }
+
+    pub fn open(
+        options: EnvOpenOptions,
+        path: impl AsRef<Path>,
+        index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static,
+        must_exit: Arc<AtomicBool>,
+    ) -> anyhow::Result<Arc<Self>> {
+        let (update_store, mut notification_receiver) = Self::new(options, path)?;
+        let update_store = Arc::new(update_store);
+
+        // Send a first notification to trigger the process.
+        let _ = update_store.notification_sender.send(());
+
+        // Init update loop to perform any pending updates at launch.
+        // Since we just launched the update store, and we still own the receiving end of the
+        // channel, this call is guaranteed to succeed.
+        update_store
+            .notification_sender
+            .try_send(())
+            .expect("Failed to init update store");
+
+        // We need a weak reference so we can take ownership on the arc later when we
+        // want to close the index.
+        let update_store_weak = Arc::downgrade(&update_store);
+        tokio::task::spawn(async move {
+            // Block and wait for something to process.
+            'outer: while notification_receiver.recv().await.is_some() {
+                loop {
+                    match update_store_weak.upgrade() {
+                        Some(update_store) => {
+                            let handler = index_handle.clone();
+                            let res = tokio::task::spawn_blocking(move || {
+                                update_store.process_pending_update(handler)
+                            })
+                            .await
+                            .expect("Fatal error processing update.");
+                            match res {
+                                Ok(Some(_)) => (),
+                                Ok(None) => break,
+                                Err(e) => {
+                                    error!("Fatal error while processing an update that requires the update store to shutdown: {}", e);
+                                    must_exit.store(true, Ordering::SeqCst);
+                                    break 'outer;
+                                }
+                            }
+                        }
+                        // the ownership on the arc has been taken, we need to exit.
+                        None => break 'outer,
+                    }
+                }
+            }
+
+            error!("Update store loop exited.");
+        });
+
+        Ok(update_store)
+    }
+
+    /// Returns the next global update id and the next update id for a given `index_uuid`.
+    fn next_update_id(&self, txn: &mut heed::RwTxn, index_uuid: Uuid) -> heed::Result<(u64, u64)> {
+        let global_id = self
+            .next_update_id
+            .get(txn, &NextIdKey::Global)?
+            .map(U64::get)
+            .unwrap_or_default();
+
+        self.next_update_id
+            .put(txn, &NextIdKey::Global, &BEU64::new(global_id + 1))?;
+
+        let update_id = self.next_update_id_raw(txn, index_uuid)?;
+
+        Ok((global_id, update_id))
+    }
+
+    /// Returns the next next update id for a given `index_uuid` without
+    /// incrementing the global update id. This is useful for the dumps.
+    fn next_update_id_raw(&self, txn: &mut heed::RwTxn, index_uuid: Uuid) -> heed::Result<u64> {
+        let update_id = self
+            .next_update_id
+            .get(txn, &NextIdKey::Index(index_uuid))?
+            .map(U64::get)
+            .unwrap_or_default();
+
+        self.next_update_id.put(
+            txn,
+            &NextIdKey::Index(index_uuid),
+            &BEU64::new(update_id + 1),
+        )?;
+
+        Ok(update_id)
+    }
+
+    /// Registers the update content in the pending store and the meta
+    /// into the pending-meta store. Returns the new unique update id.
+    pub fn register_update(
+        &self,
+        meta: UpdateMeta,
+        content: Option<Uuid>,
+        index_uuid: Uuid,
+    ) -> heed::Result<Enqueued> {
+        let mut txn = self.env.write_txn()?;
+
+        let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?;
+        let meta = Enqueued::new(meta, update_id, content);
+
+        self.pending_queue
+            .put(&mut txn, &(global_id, index_uuid, update_id), &meta)?;
+
+        txn.commit()?;
+
+        self.notification_sender
+            .blocking_send(())
+            .expect("Update store loop exited.");
+        Ok(meta)
+    }
+
+    /// Push already processed update in the UpdateStore without triggering the notification
+    /// process. This is useful for the dumps.
+    pub fn register_raw_updates(
+        &self,
+        wtxn: &mut heed::RwTxn,
+        update: &UpdateStatus,
+        index_uuid: Uuid,
+    ) -> heed::Result<()> {
+        match update {
+            UpdateStatus::Enqueued(enqueued) => {
+                let (global_id, _update_id) = self.next_update_id(wtxn, index_uuid)?;
+                self.pending_queue.remap_key_type::<PendingKeyCodec>().put(
+                    wtxn,
+                    &(global_id, index_uuid, enqueued.id()),
+                    &enqueued,
+                )?;
+            }
+            _ => {
+                let _update_id = self.next_update_id_raw(wtxn, index_uuid)?;
+                self.updates
+                    .put(wtxn, &(index_uuid, update.id()), &update)?;
+            }
+        }
+        Ok(())
+    }
+
+    /// Executes the user provided function on the next pending update (the one with the lowest id).
+    /// This is asynchronous as it let the user process the update with a read-only txn and
+    /// only writing the result meta to the processed-meta store *after* it has been processed.
+    fn process_pending_update(&self, index_handle: impl IndexActorHandle) -> Result<Option<()>> {
+        // Create a read transaction to be able to retrieve the pending update in order.
+        let rtxn = self.env.read_txn()?;
+        let first_meta = self.pending_queue.first(&rtxn)?;
+        drop(rtxn);
+
+        // If there is a pending update we process and only keep
+        // a reader while processing it, not a writer.
+        match first_meta {
+            Some(((global_id, index_uuid, _), mut pending)) => {
+                let content = pending.content.take();
+                let processing = pending.processing();
+                // Acquire the state lock and set the current state to processing.
+                // txn must *always* be acquired after state lock, or it will dead lock.
+                let state = self.state.write();
+                state.swap(State::Processing(index_uuid, processing.clone()));
+
+                let result =
+                    self.perform_update(content, processing, index_handle, index_uuid, global_id);
+
+                state.swap(State::Idle);
+
+                result
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn perform_update(
+        &self,
+        content: Option<Uuid>,
+        processing: Processing,
+        index_handle: impl IndexActorHandle,
+        index_uuid: Uuid,
+        global_id: u64,
+    ) -> Result<Option<()>> {
+        let content_path = content.map(|uuid| update_uuid_to_file_path(&self.path, uuid));
+        let update_id = processing.id();
+
+        let file = match content_path {
+            Some(ref path) => {
+                let file = File::open(path)?;
+                Some(file)
+            }
+            None => None,
+        };
+
+        // Process the pending update using the provided user function.
+        let handle = Handle::current();
+        let result =
+            match handle.block_on(index_handle.update(index_uuid, processing.clone(), file)) {
+                Ok(result) => result,
+                Err(e) => Err(processing.fail(e.into())),
+            };
+
+        // Once the pending update have been successfully processed
+        // we must remove the content from the pending and processing stores and
+        // write the *new* meta to the processed-meta store and commit.
+        let mut wtxn = self.env.write_txn()?;
+        self.pending_queue
+            .delete(&mut wtxn, &(global_id, index_uuid, update_id))?;
+
+        let result = match result {
+            Ok(res) => res.into(),
+            Err(res) => res.into(),
+        };
+
+        self.updates
+            .put(&mut wtxn, &(index_uuid, update_id), &result)?;
+
+        wtxn.commit()?;
+
+        if let Some(ref path) = content_path {
+            remove_file(&path)?;
+        }
+
+        Ok(Some(()))
+    }
+
+    /// List the updates for `index_uuid`.
+    pub fn list(&self, index_uuid: Uuid) -> Result<Vec<UpdateStatus>> {
+        let mut update_list = BTreeMap::<u64, UpdateStatus>::new();
+
+        let txn = self.env.read_txn()?;
+
+        let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data();
+        for entry in pendings {
+            let ((_, uuid, id), pending) = entry?;
+            if uuid == index_uuid {
+                update_list.insert(id, pending.decode()?.into());
+            }
+        }
+
+        let updates = self
+            .updates
+            .remap_key_type::<ByteSlice>()
+            .prefix_iter(&txn, index_uuid.as_bytes())?;
+
+        for entry in updates {
+            let (_, update) = entry?;
+            update_list.insert(update.id(), update);
+        }
+
+        // If the currently processing update is from this index, replace the corresponding pending update with this one.
+        match *self.state.read() {
+            State::Processing(uuid, ref processing) if uuid == index_uuid => {
+                update_list.insert(processing.id(), processing.clone().into());
+            }
+            _ => (),
+        }
+
+        Ok(update_list.into_iter().map(|(_, v)| v).collect())
+    }
+
+    /// Returns the update associated meta or `None` if the update doesn't exist.
+    pub fn meta(&self, index_uuid: Uuid, update_id: u64) -> heed::Result<Option<UpdateStatus>> {
+        // Check if the update is the one currently processing
+        match *self.state.read() {
+            State::Processing(uuid, ref processing)
+                if uuid == index_uuid && processing.id() == update_id =>
+            {
+                return Ok(Some(processing.clone().into()));
+            }
+            _ => (),
+        }
+
+        let txn = self.env.read_txn()?;
+        // Else, check if it is in the updates database:
+        let update = self.updates.get(&txn, &(index_uuid, update_id))?;
+
+        if let Some(update) = update {
+            return Ok(Some(update));
+        }
+
+        // If nothing was found yet, we resolve to iterate over the pending queue.
+        let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data();
+
+        for entry in pendings {
+            let ((_, uuid, id), pending) = entry?;
+            if uuid == index_uuid && id == update_id {
+                return Ok(Some(pending.decode()?.into()));
+            }
+        }
+
+        // No update was found.
+        Ok(None)
+    }
+
+    /// Delete all updates for an index from the update store. If the currently processing update
+    /// is for `index_uuid`, the call will block until the update is terminated.
+    pub fn delete_all(&self, index_uuid: Uuid) -> Result<()> {
+        let mut txn = self.env.write_txn()?;
+        // Contains all the content file paths that we need to be removed if the deletion was successful.
+        let mut uuids_to_remove = Vec::new();
+
+        let mut pendings = self.pending_queue.iter_mut(&mut txn)?.lazily_decode_data();
+
+        while let Some(Ok(((_, uuid, _), pending))) = pendings.next() {
+            if uuid == index_uuid {
+                unsafe {
+                    pendings.del_current()?;
+                }
+                let mut pending = pending.decode()?;
+                if let Some(update_uuid) = pending.content.take() {
+                    uuids_to_remove.push(update_uuid);
+                }
+            }
+        }
+
+        drop(pendings);
+
+        let mut updates = self
+            .updates
+            .remap_key_type::<ByteSlice>()
+            .prefix_iter_mut(&mut txn, index_uuid.as_bytes())?
+            .lazily_decode_data();
+
+        while let Some(_) = updates.next() {
+            unsafe {
+                updates.del_current()?;
+            }
+        }
+
+        drop(updates);
+
+        txn.commit()?;
+
+        uuids_to_remove
+            .iter()
+            .map(|uuid| update_uuid_to_file_path(&self.path, *uuid))
+            .for_each(|path| {
+                let _ = remove_file(path);
+            });
+
+        // If the currently processing update is from our index, we wait until it is
+        // finished before returning. This ensure that no write to the index occurs after we delete it.
+        if let State::Processing(uuid, _) = *self.state.read() {
+            if uuid == index_uuid {
+                // wait for a write lock, do nothing with it.
+                self.state.write();
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn snapshot(
+        &self,
+        uuids: &HashSet<Uuid>,
+        path: impl AsRef<Path>,
+        handle: impl IndexActorHandle + Clone,
+    ) -> Result<()> {
+        let state_lock = self.state.write();
+        state_lock.swap(State::Snapshoting);
+
+        let txn = self.env.write_txn()?;
+
+        let update_path = path.as_ref().join("updates");
+        create_dir_all(&update_path)?;
+
+        // acquire write lock to prevent further writes during snapshot
+        create_dir_all(&update_path)?;
+        let db_path = update_path.join("data.mdb");
+
+        // create db snapshot
+        self.env.copy_to_path(&db_path, CompactionOption::Enabled)?;
+
+        let update_files_path = update_path.join(UPDATE_DIR);
+        create_dir_all(&update_files_path)?;
+
+        let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data();
+
+        for entry in pendings {
+            let ((_, uuid, _), pending) = entry?;
+            if uuids.contains(&uuid) {
+                if let Enqueued {
+                    content: Some(uuid),
+                    ..
+                } = pending.decode()?
+                {
+                    let path = update_uuid_to_file_path(&self.path, uuid);
+                    copy(path, &update_files_path)?;
+                }
+            }
+        }
+
+        let path = &path.as_ref().to_path_buf();
+        let handle = &handle;
+        // Perform the snapshot of each index concurently. Only a third of the capabilities of
+        // the index actor at a time not to put too much pressure on the index actor
+        let mut stream = futures::stream::iter(uuids.iter())
+            .map(move |uuid| handle.snapshot(*uuid, path.clone()))
+            .buffer_unordered(CONCURRENT_INDEX_MSG / 3);
+
+        Handle::current().block_on(async {
+            while let Some(res) = stream.next().await {
+                res?;
+            }
+            Ok(()) as Result<()>
+        })?;
+
+        Ok(())
+    }
+
+    pub fn get_info(&self) -> Result<UpdateStoreInfo> {
+        let mut size = self.env.size();
+        let txn = self.env.read_txn()?;
+        for entry in self.pending_queue.iter(&txn)? {
+            let (_, pending) = entry?;
+            if let Enqueued {
+                content: Some(uuid),
+                ..
+            } = pending
+            {
+                let path = update_uuid_to_file_path(&self.path, uuid);
+                size += File::open(path)?.metadata()?.len();
+            }
+        }
+        let processing = match *self.state.read() {
+            State::Processing(uuid, _) => Some(uuid),
+            _ => None,
+        };
+
+        Ok(UpdateStoreInfo { size, processing })
+    }
+}
+
+fn update_uuid_to_file_path(root: impl AsRef<Path>, uuid: Uuid) -> PathBuf {
+    root.as_ref()
+        .join(UPDATE_DIR)
+        .join(format!("update_{}", uuid))
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::index_controller::{
+        index_actor::{error::IndexActorError, MockIndexActorHandle},
+        UpdateResult,
+    };
+
+    use futures::future::ok;
+
+    #[actix_rt::test]
+    async fn test_next_id() {
+        let dir = tempfile::tempdir_in(".").unwrap();
+        let mut options = EnvOpenOptions::new();
+        let handle = Arc::new(MockIndexActorHandle::new());
+        options.map_size(4096 * 100);
+        let update_store = UpdateStore::open(
+            options,
+            dir.path(),
+            handle,
+            Arc::new(AtomicBool::new(false)),
+        )
+        .unwrap();
+
+        let index1_uuid = Uuid::new_v4();
+        let index2_uuid = Uuid::new_v4();
+
+        let mut txn = update_store.env.write_txn().unwrap();
+        let ids = update_store.next_update_id(&mut txn, index1_uuid).unwrap();
+        txn.commit().unwrap();
+        assert_eq!((0, 0), ids);
+
+        let mut txn = update_store.env.write_txn().unwrap();
+        let ids = update_store.next_update_id(&mut txn, index2_uuid).unwrap();
+        txn.commit().unwrap();
+        assert_eq!((1, 0), ids);
+
+        let mut txn = update_store.env.write_txn().unwrap();
+        let ids = update_store.next_update_id(&mut txn, index1_uuid).unwrap();
+        txn.commit().unwrap();
+        assert_eq!((2, 1), ids);
+    }
+
+    #[actix_rt::test]
+    async fn test_register_update() {
+        let dir = tempfile::tempdir_in(".").unwrap();
+        let mut options = EnvOpenOptions::new();
+        let handle = Arc::new(MockIndexActorHandle::new());
+        options.map_size(4096 * 100);
+        let update_store = UpdateStore::open(
+            options,
+            dir.path(),
+            handle,
+            Arc::new(AtomicBool::new(false)),
+        )
+        .unwrap();
+        let meta = UpdateMeta::ClearDocuments;
+        let uuid = Uuid::new_v4();
+        let store_clone = update_store.clone();
+        tokio::task::spawn_blocking(move || {
+            store_clone.register_update(meta, None, uuid).unwrap();
+        })
+        .await
+        .unwrap();
+
+        let txn = update_store.env.read_txn().unwrap();
+        assert!(update_store
+            .pending_queue
+            .get(&txn, &(0, uuid, 0))
+            .unwrap()
+            .is_some());
+    }
+
+    #[actix_rt::test]
+    async fn test_process_update() {
+        let dir = tempfile::tempdir_in(".").unwrap();
+        let mut handle = MockIndexActorHandle::new();
+
+        handle
+            .expect_update()
+            .times(2)
+            .returning(|_index_uuid, processing, _file| {
+                if processing.id() == 0 {
+                    Box::pin(ok(Ok(processing.process(UpdateResult::Other))))
+                } else {
+                    Box::pin(ok(Err(
+                        processing.fail(IndexActorError::ExistingPrimaryKey.into())
+                    )))
+                }
+            });
+
+        let handle = Arc::new(handle);
+
+        let mut options = EnvOpenOptions::new();
+        options.map_size(4096 * 100);
+        let store = UpdateStore::open(
+            options,
+            dir.path(),
+            handle.clone(),
+            Arc::new(AtomicBool::new(false)),
+        )
+        .unwrap();
+
+        // wait a bit for the event loop exit.
+        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+
+        let mut txn = store.env.write_txn().unwrap();
+
+        let update = Enqueued::new(UpdateMeta::ClearDocuments, 0, None);
+        let uuid = Uuid::new_v4();
+
+        store
+            .pending_queue
+            .put(&mut txn, &(0, uuid, 0), &update)
+            .unwrap();
+
+        let update = Enqueued::new(UpdateMeta::ClearDocuments, 1, None);
+
+        store
+            .pending_queue
+            .put(&mut txn, &(1, uuid, 1), &update)
+            .unwrap();
+
+        txn.commit().unwrap();
+
+        // Process the pending, and check that it has been moved to the update databases, and
+        // removed from the pending database.
+        let store_clone = store.clone();
+        tokio::task::spawn_blocking(move || {
+            store_clone.process_pending_update(handle.clone()).unwrap();
+            store_clone.process_pending_update(handle).unwrap();
+        })
+        .await
+        .unwrap();
+
+        let txn = store.env.read_txn().unwrap();
+
+        assert!(store.pending_queue.first(&txn).unwrap().is_none());
+        let update = store.updates.get(&txn, &(uuid, 0)).unwrap().unwrap();
+
+        assert!(matches!(update, UpdateStatus::Processed(_)));
+        let update = store.updates.get(&txn, &(uuid, 1)).unwrap().unwrap();
+
+        assert!(matches!(update, UpdateStatus::Failed(_)));
+    }
+}

meilisearch-http/src/index_controller/updates.rs

@@ -0,0 +1,233 @@
+use chrono::{DateTime, Utc};
+use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use crate::{
+    error::ResponseError,
+    index::{Settings, Unchecked},
+};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum UpdateResult {
+    DocumentsAddition(DocumentAdditionResult),
+    DocumentDeletion { deleted: u64 },
+    Other,
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum UpdateMeta {
+    DocumentsAddition {
+        method: IndexDocumentsMethod,
+        format: UpdateFormat,
+        primary_key: Option<String>,
+    },
+    ClearDocuments,
+    DeleteDocuments {
+        ids: Vec<String>,
+    },
+    Settings(Settings<Unchecked>),
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Enqueued {
+    pub update_id: u64,
+    pub meta: UpdateMeta,
+    pub enqueued_at: DateTime<Utc>,
+    pub content: Option<Uuid>,
+}
+
+impl Enqueued {
+    pub fn new(meta: UpdateMeta, update_id: u64, content: Option<Uuid>) -> Self {
+        Self {
+            enqueued_at: Utc::now(),
+            meta,
+            update_id,
+            content,
+        }
+    }
+
+    pub fn processing(self) -> Processing {
+        Processing {
+            from: self,
+            started_processing_at: Utc::now(),
+        }
+    }
+
+    pub fn abort(self) -> Aborted {
+        Aborted {
+            from: self,
+            aborted_at: Utc::now(),
+        }
+    }
+
+    pub fn meta(&self) -> &UpdateMeta {
+        &self.meta
+    }
+
+    pub fn id(&self) -> u64 {
+        self.update_id
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Processed {
+    pub success: UpdateResult,
+    pub processed_at: DateTime<Utc>,
+    #[serde(flatten)]
+    pub from: Processing,
+}
+
+impl Processed {
+    pub fn id(&self) -> u64 {
+        self.from.id()
+    }
+
+    pub fn meta(&self) -> &UpdateMeta {
+        self.from.meta()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Processing {
+    #[serde(flatten)]
+    pub from: Enqueued,
+    pub started_processing_at: DateTime<Utc>,
+}
+
+impl Processing {
+    pub fn id(&self) -> u64 {
+        self.from.id()
+    }
+
+    pub fn meta(&self) -> &UpdateMeta {
+        self.from.meta()
+    }
+
+    pub fn process(self, success: UpdateResult) -> Processed {
+        Processed {
+            success,
+            from: self,
+            processed_at: Utc::now(),
+        }
+    }
+
+    pub fn fail(self, error: ResponseError) -> Failed {
+        Failed {
+            from: self,
+            error,
+            failed_at: Utc::now(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Aborted {
+    #[serde(flatten)]
+    from: Enqueued,
+    aborted_at: DateTime<Utc>,
+}
+
+impl Aborted {
+    pub fn id(&self) -> u64 {
+        self.from.id()
+    }
+
+    pub fn meta(&self) -> &UpdateMeta {
+        self.from.meta()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Failed {
+    #[serde(flatten)]
+    pub from: Processing,
+    pub error: ResponseError,
+    pub failed_at: DateTime<Utc>,
+}
+
+impl Failed {
+    pub fn id(&self) -> u64 {
+        self.from.id()
+    }
+
+    pub fn meta(&self) -> &UpdateMeta {
+        self.from.meta()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "status", rename_all = "camelCase")]
+pub enum UpdateStatus {
+    Processing(Processing),
+    Enqueued(Enqueued),
+    Processed(Processed),
+    Aborted(Aborted),
+    Failed(Failed),
+}
+
+impl UpdateStatus {
+    pub fn id(&self) -> u64 {
+        match self {
+            UpdateStatus::Processing(u) => u.id(),
+            UpdateStatus::Enqueued(u) => u.id(),
+            UpdateStatus::Processed(u) => u.id(),
+            UpdateStatus::Aborted(u) => u.id(),
+            UpdateStatus::Failed(u) => u.id(),
+        }
+    }
+
+    pub fn meta(&self) -> &UpdateMeta {
+        match self {
+            UpdateStatus::Processing(u) => u.meta(),
+            UpdateStatus::Enqueued(u) => u.meta(),
+            UpdateStatus::Processed(u) => u.meta(),
+            UpdateStatus::Aborted(u) => u.meta(),
+            UpdateStatus::Failed(u) => u.meta(),
+        }
+    }
+
+    pub fn processed(&self) -> Option<&Processed> {
+        match self {
+            UpdateStatus::Processed(p) => Some(p),
+            _ => None,
+        }
+    }
+}
+
+impl From<Enqueued> for UpdateStatus {
+    fn from(other: Enqueued) -> Self {
+        Self::Enqueued(other)
+    }
+}
+
+impl From<Aborted> for UpdateStatus {
+    fn from(other: Aborted) -> Self {
+        Self::Aborted(other)
+    }
+}
+
+impl From<Processed> for UpdateStatus {
+    fn from(other: Processed) -> Self {
+        Self::Processed(other)
+    }
+}
+
+impl From<Processing> for UpdateStatus {
+    fn from(other: Processing) -> Self {
+        Self::Processing(other)
+    }
+}
+
+impl From<Failed> for UpdateStatus {
+    fn from(other: Failed) -> Self {
+        Self::Failed(other)
+    }
+}

meilisearch-http/src/index_controller/uuid_resolver/actor.rs

@@ -0,0 +1,98 @@
+use std::{collections::HashSet, path::PathBuf};
+
+use log::{trace, warn};
+use tokio::sync::mpsc;
+use uuid::Uuid;
+
+use super::{error::UuidResolverError, Result, UuidResolveMsg, UuidStore};
+
+pub struct UuidResolverActor<S> {
+    inbox: mpsc::Receiver<UuidResolveMsg>,
+    store: S,
+}
+
+impl<S: UuidStore> UuidResolverActor<S> {
+    pub fn new(inbox: mpsc::Receiver<UuidResolveMsg>, store: S) -> Self {
+        Self { inbox, store }
+    }
+
+    pub async fn run(mut self) {
+        use UuidResolveMsg::*;
+
+        trace!("uuid resolver started");
+
+        loop {
+            match self.inbox.recv().await {
+                Some(Get { uid: name, ret }) => {
+                    let _ = ret.send(self.handle_get(name).await);
+                }
+                Some(Delete { uid: name, ret }) => {
+                    let _ = ret.send(self.handle_delete(name).await);
+                }
+                Some(List { ret }) => {
+                    let _ = ret.send(self.handle_list().await);
+                }
+                Some(Insert { ret, uuid, name }) => {
+                    let _ = ret.send(self.handle_insert(name, uuid).await);
+                }
+                Some(SnapshotRequest { path, ret }) => {
+                    let _ = ret.send(self.handle_snapshot(path).await);
+                }
+                Some(GetSize { ret }) => {
+                    let _ = ret.send(self.handle_get_size().await);
+                }
+                Some(DumpRequest { path, ret }) => {
+                    let _ = ret.send(self.handle_dump(path).await);
+                }
+                // all senders have been dropped, need to quit.
+                None => break,
+            }
+        }
+
+        warn!("exiting uuid resolver loop");
+    }
+
+    async fn handle_get(&self, uid: String) -> Result<Uuid> {
+        self.store
+            .get_uuid(uid.clone())
+            .await?
+            .ok_or(UuidResolverError::UnexistingIndex(uid))
+    }
+
+    async fn handle_delete(&self, uid: String) -> Result<Uuid> {
+        self.store
+            .delete(uid.clone())
+            .await?
+            .ok_or(UuidResolverError::UnexistingIndex(uid))
+    }
+
+    async fn handle_list(&self) -> Result<Vec<(String, Uuid)>> {
+        let result = self.store.list().await?;
+        Ok(result)
+    }
+
+    async fn handle_snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
+        self.store.snapshot(path).await
+    }
+
+    async fn handle_dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
+        self.store.dump(path).await
+    }
+
+    async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> {
+        if !is_index_uid_valid(&uid) {
+            return Err(UuidResolverError::BadlyFormatted(uid));
+        }
+        self.store.insert(uid, uuid).await?;
+        Ok(())
+    }
+
+    async fn handle_get_size(&self) -> Result<u64> {
+        self.store.get_size().await
+    }
+}
+
+fn is_index_uid_valid(uid: &str) -> bool {
+    uid.chars()
+        .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
+}

meilisearch-http/src/index_controller/uuid_resolver/error.rs

@@ -0,0 +1,34 @@
+use meilisearch_error::{Code, ErrorCode};
+
+pub type Result<T> = std::result::Result<T, UuidResolverError>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum UuidResolverError {
+    #[error("Index already exists.")]
+    NameAlreadyExist,
+    #[error("Index \"{0}\" not found.")]
+    UnexistingIndex(String),
+    #[error("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")]
+    BadlyFormatted(String),
+    #[error("Internal error: {0}")]
+    Internal(Box<dyn std::error::Error + Sync + Send + 'static>),
+}
+
+internal_error!(
+    UuidResolverError: heed::Error,
+    uuid::Error,
+    std::io::Error,
+    tokio::task::JoinError,
+    serde_json::Error
+);
+
+impl ErrorCode for UuidResolverError {
+    fn error_code(&self) -> Code {
+        match self {
+            UuidResolverError::NameAlreadyExist => Code::IndexAlreadyExists,
+            UuidResolverError::UnexistingIndex(_) => Code::IndexNotFound,
+            UuidResolverError::BadlyFormatted(_) => Code::InvalidIndexUid,
+            UuidResolverError::Internal(_) => Code::Internal,
+        }
+    }
+}

meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs

@@ -0,0 +1,87 @@
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+
+use tokio::sync::{mpsc, oneshot};
+use uuid::Uuid;
+
+use super::{HeedUuidStore, Result, UuidResolveMsg, UuidResolverActor, UuidResolverHandle};
+
+#[derive(Clone)]
+pub struct UuidResolverHandleImpl {
+    sender: mpsc::Sender<UuidResolveMsg>,
+}
+
+impl UuidResolverHandleImpl {
+    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
+        let (sender, reveiver) = mpsc::channel(100);
+        let store = HeedUuidStore::new(path)?;
+        let actor = UuidResolverActor::new(reveiver, store);
+        tokio::spawn(actor.run());
+        Ok(Self { sender })
+    }
+}
+
+#[async_trait::async_trait]
+impl UuidResolverHandle for UuidResolverHandleImpl {
+    async fn get(&self, name: String) -> Result<Uuid> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UuidResolveMsg::Get { uid: name, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver
+            .await
+            .expect("Uuid resolver actor has been killed")?)
+    }
+
+    async fn delete(&self, name: String) -> Result<Uuid> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UuidResolveMsg::Delete { uid: name, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver
+            .await
+            .expect("Uuid resolver actor has been killed")?)
+    }
+
+    async fn list(&self) -> Result<Vec<(String, Uuid)>> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UuidResolveMsg::List { ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver
+            .await
+            .expect("Uuid resolver actor has been killed")?)
+    }
+
+    async fn insert(&self, name: String, uuid: Uuid) -> Result<()> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UuidResolveMsg::Insert { ret, name, uuid };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver
+            .await
+            .expect("Uuid resolver actor has been killed")?)
+    }
+
+    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UuidResolveMsg::SnapshotRequest { path, ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver
+            .await
+            .expect("Uuid resolver actor has been killed")?)
+    }
+
+    async fn get_size(&self) -> Result<u64> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UuidResolveMsg::GetSize { ret };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver
+            .await
+            .expect("Uuid resolver actor has been killed")?)
+    }
+    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
+        let (ret, receiver) = oneshot::channel();
+        let msg = UuidResolveMsg::DumpRequest { ret, path };
+        let _ = self.sender.send(msg).await;
+        Ok(receiver
+            .await
+            .expect("Uuid resolver actor has been killed")?)
+    }
+}

meilisearch-http/src/index_controller/uuid_resolver/message.rs

@@ -0,0 +1,37 @@
+use std::collections::HashSet;
+use std::path::PathBuf;
+
+use tokio::sync::oneshot;
+use uuid::Uuid;
+
+use super::Result;
+
+pub enum UuidResolveMsg {
+    Get {
+        uid: String,
+        ret: oneshot::Sender<Result<Uuid>>,
+    },
+    Delete {
+        uid: String,
+        ret: oneshot::Sender<Result<Uuid>>,
+    },
+    List {
+        ret: oneshot::Sender<Result<Vec<(String, Uuid)>>>,
+    },
+    Insert {
+        uuid: Uuid,
+        name: String,
+        ret: oneshot::Sender<Result<()>>,
+    },
+    SnapshotRequest {
+        path: PathBuf,
+        ret: oneshot::Sender<Result<HashSet<Uuid>>>,
+    },
+    GetSize {
+        ret: oneshot::Sender<Result<u64>>,
+    },
+    DumpRequest {
+        path: PathBuf,
+        ret: oneshot::Sender<Result<HashSet<Uuid>>>,
+    },
+}

meilisearch-http/src/index_controller/uuid_resolver/mod.rs

@@ -0,0 +1,35 @@
+mod actor;
+pub mod error;
+mod handle_impl;
+mod message;
+pub mod store;
+
+use std::collections::HashSet;
+use std::path::PathBuf;
+
+use uuid::Uuid;
+
+use actor::UuidResolverActor;
+use error::Result;
+use message::UuidResolveMsg;
+use store::UuidStore;
+
+#[cfg(test)]
+use mockall::automock;
+
+pub use handle_impl::UuidResolverHandleImpl;
+pub use store::HeedUuidStore;
+
+const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB
+
+#[async_trait::async_trait]
+#[cfg_attr(test, automock)]
+pub trait UuidResolverHandle {
+    async fn get(&self, name: String) -> Result<Uuid>;
+    async fn insert(&self, name: String, uuid: Uuid) -> Result<()>;
+    async fn delete(&self, name: String) -> Result<Uuid>;
+    async fn list(&self) -> Result<Vec<(String, Uuid)>>;
+    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
+    async fn get_size(&self) -> Result<u64>;
+    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
+}

meilisearch-http/src/index_controller/uuid_resolver/store.rs

@@ -0,0 +1,224 @@
+use std::collections::HashSet;
+use std::fs::{create_dir_all, File};
+use std::io::{BufRead, BufReader, Write};
+use std::path::{Path, PathBuf};
+
+use heed::types::{ByteSlice, Str};
+use heed::{CompactionOption, Database, Env, EnvOpenOptions};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use super::{error::UuidResolverError, Result, UUID_STORE_SIZE};
+use crate::helpers::EnvSizer;
+
+#[derive(Serialize, Deserialize)]
+struct DumpEntry {
+    uuid: Uuid,
+    uid: String,
+}
+
+const UUIDS_DB_PATH: &str = "index_uuids";
+
+#[async_trait::async_trait]
+pub trait UuidStore: Sized {
+    // Create a new entry for `name`. Return an error if `err` and the entry already exists, return
+    // the uuid otherwise.
+    async fn get_uuid(&self, uid: String) -> Result<Option<Uuid>>;
+    async fn delete(&self, uid: String) -> Result<Option<Uuid>>;
+    async fn list(&self) -> Result<Vec<(String, Uuid)>>;
+    async fn insert(&self, name: String, uuid: Uuid) -> Result<()>;
+    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
+    async fn get_size(&self) -> Result<u64>;
+    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
+}
+
+#[derive(Clone)]
+pub struct HeedUuidStore {
+    env: Env,
+    db: Database<Str, ByteSlice>,
+}
+
+impl HeedUuidStore {
+    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
+        let path = path.as_ref().join(UUIDS_DB_PATH);
+        create_dir_all(&path)?;
+        let mut options = EnvOpenOptions::new();
+        options.map_size(UUID_STORE_SIZE); // 1GB
+        let env = options.open(path)?;
+        let db = env.create_database(None)?;
+        Ok(Self { env, db })
+    }
+
+    pub fn get_uuid(&self, name: String) -> Result<Option<Uuid>> {
+        let env = self.env.clone();
+        let db = self.db;
+        let txn = env.read_txn()?;
+        match db.get(&txn, &name)? {
+            Some(uuid) => {
+                let uuid = Uuid::from_slice(uuid)?;
+                Ok(Some(uuid))
+            }
+            None => Ok(None),
+        }
+    }
+
+    pub fn delete(&self, uid: String) -> Result<Option<Uuid>> {
+        let env = self.env.clone();
+        let db = self.db;
+        let mut txn = env.write_txn()?;
+        match db.get(&txn, &uid)? {
+            Some(uuid) => {
+                let uuid = Uuid::from_slice(uuid)?;
+                db.delete(&mut txn, &uid)?;
+                txn.commit()?;
+                Ok(Some(uuid))
+            }
+            None => Ok(None),
+        }
+    }
+
+    pub fn list(&self) -> Result<Vec<(String, Uuid)>> {
+        let env = self.env.clone();
+        let db = self.db;
+        let txn = env.read_txn()?;
+        let mut entries = Vec::new();
+        for entry in db.iter(&txn)? {
+            let (name, uuid) = entry?;
+            let uuid = Uuid::from_slice(uuid)?;
+            entries.push((name.to_owned(), uuid))
+        }
+        Ok(entries)
+    }
+
+    pub fn insert(&self, name: String, uuid: Uuid) -> Result<()> {
+        let env = self.env.clone();
+        let db = self.db;
+        let mut txn = env.write_txn()?;
+
+        if db.get(&txn, &name)?.is_some() {
+            return Err(UuidResolverError::NameAlreadyExist);
+        }
+
+        db.put(&mut txn, &name, uuid.as_bytes())?;
+        txn.commit()?;
+        Ok(())
+    }
+
+    pub fn snapshot(&self, mut path: PathBuf) -> Result<HashSet<Uuid>> {
+        let env = self.env.clone();
+        let db = self.db;
+        // Write transaction to acquire a lock on the database.
+        let txn = env.write_txn()?;
+        let mut entries = HashSet::new();
+        for entry in db.iter(&txn)? {
+            let (_, uuid) = entry?;
+            let uuid = Uuid::from_slice(uuid)?;
+            entries.insert(uuid);
+        }
+
+        // only perform snapshot if there are indexes
+        if !entries.is_empty() {
+            path.push(UUIDS_DB_PATH);
+            create_dir_all(&path).unwrap();
+            path.push("data.mdb");
+            env.copy_to_path(path, CompactionOption::Enabled)?;
+        }
+        Ok(entries)
+    }
+
+    pub fn get_size(&self) -> Result<u64> {
+        Ok(self.env.size())
+    }
+
+    pub fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
+        let dump_path = path.join(UUIDS_DB_PATH);
+        create_dir_all(&dump_path)?;
+        let dump_file_path = dump_path.join("data.jsonl");
+        let mut dump_file = File::create(&dump_file_path)?;
+        let mut uuids = HashSet::new();
+
+        let txn = self.env.read_txn()?;
+        for entry in self.db.iter(&txn)? {
+            let (uid, uuid) = entry?;
+            let uid = uid.to_string();
+            let uuid = Uuid::from_slice(uuid)?;
+
+            let entry = DumpEntry { uuid, uid };
+            serde_json::to_writer(&mut dump_file, &entry)?;
+            dump_file.write_all(b"\n").unwrap();
+
+            uuids.insert(uuid);
+        }
+
+        Ok(uuids)
+    }
+
+    pub fn load_dump(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> Result<()> {
+        let uuid_resolver_path = dst.as_ref().join(UUIDS_DB_PATH);
+        std::fs::create_dir_all(&uuid_resolver_path)?;
+
+        let src_indexes = src.as_ref().join(UUIDS_DB_PATH).join("data.jsonl");
+        let indexes = File::open(&src_indexes)?;
+        let mut indexes = BufReader::new(indexes);
+        let mut line = String::new();
+
+        let db = Self::new(dst)?;
+        let mut txn = db.env.write_txn()?;
+
+        loop {
+            match indexes.read_line(&mut line) {
+                Ok(0) => break,
+                Ok(_) => {
+                    let DumpEntry { uuid, uid } = serde_json::from_str(&line)?;
+                    println!("importing {} {}", uid, uuid);
+                    db.db.put(&mut txn, &uid, uuid.as_bytes())?;
+                }
+                Err(e) => return Err(e.into()),
+            }
+
+            line.clear();
+        }
+        txn.commit()?;
+
+        db.env.prepare_for_closing().wait();
+
+        Ok(())
+    }
+}
+
+#[async_trait::async_trait]
+impl UuidStore for HeedUuidStore {
+    async fn get_uuid(&self, name: String) -> Result<Option<Uuid>> {
+        let this = self.clone();
+        tokio::task::spawn_blocking(move || this.get_uuid(name)).await?
+    }
+
+    async fn delete(&self, uid: String) -> Result<Option<Uuid>> {
+        let this = self.clone();
+        tokio::task::spawn_blocking(move || this.delete(uid)).await?
+    }
+
+    async fn list(&self) -> Result<Vec<(String, Uuid)>> {
+        let this = self.clone();
+        tokio::task::spawn_blocking(move || this.list()).await?
+    }
+
+    async fn insert(&self, name: String, uuid: Uuid) -> Result<()> {
+        let this = self.clone();
+        tokio::task::spawn_blocking(move || this.insert(name, uuid)).await?
+    }
+
+    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
+        let this = self.clone();
+        tokio::task::spawn_blocking(move || this.snapshot(path)).await?
+    }
+
+    async fn get_size(&self) -> Result<u64> {
+        self.get_size()
+    }
+
+    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
+        let this = self.clone();
+        tokio::task::spawn_blocking(move || this.dump(path)).await?
+    }
+}

meilisearch-http/src/lib.rs

@@ -0,0 +1,138 @@
+pub mod data;
+#[macro_use]
+pub mod error;
+#[macro_use]
+pub mod extractors;
+pub mod helpers;
+mod index;
+mod index_controller;
+pub mod option;
+pub mod routes;
+
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+pub mod analytics;
+
+use crate::extractors::authentication::AuthConfig;
+
+pub use self::data::Data;
+pub use option::Opt;
+
+use actix_web::web;
+
+use extractors::authentication::policies::*;
+use extractors::payload::PayloadConfig;
+
+pub fn configure_data(config: &mut web::ServiceConfig, data: Data) {
+    let http_payload_size_limit = data.http_payload_size_limit();
+    config
+        .data(data.clone())
+        .app_data(data)
+        .app_data(
+            web::JsonConfig::default()
+                .limit(http_payload_size_limit)
+                .content_type(|_mime| true) // Accept all mime types
+                .error_handler(|err, _req| error::payload_error_handler(err).into()),
+        )
+        .app_data(PayloadConfig::new(http_payload_size_limit))
+        .app_data(
+            web::QueryConfig::default()
+                .error_handler(|err, _req| error::payload_error_handler(err).into()),
+        );
+}
+
+pub fn configure_auth(config: &mut web::ServiceConfig, data: &Data) {
+    let keys = data.api_keys();
+    let auth_config = if let Some(ref master_key) = keys.master {
+        let private_key = keys.private.as_ref().unwrap();
+        let public_key = keys.public.as_ref().unwrap();
+        let mut policies = init_policies!(Public, Private, Admin);
+        create_users!(
+            policies,
+            master_key.as_bytes() => { Admin, Private, Public },
+            private_key.as_bytes() => { Private, Public },
+            public_key.as_bytes() => { Public }
+        );
+        AuthConfig::Auth(policies)
+    } else {
+        AuthConfig::NoAuth
+    };
+
+    config.app_data(auth_config);
+}
+
+#[cfg(feature = "mini-dashboard")]
+pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) {
+    use actix_web::HttpResponse;
+    use actix_web_static_files::Resource;
+
+    mod generated {
+        include!(concat!(env!("OUT_DIR"), "/generated.rs"));
+    }
+
+    if enable_frontend {
+        let generated = generated::generate();
+        let mut scope = web::scope("/");
+        // Generate routes for mini-dashboard assets
+        for (path, resource) in generated.into_iter() {
+            let Resource {
+                mime_type, data, ..
+            } = resource;
+            // Redirect index.html to /
+            if path == "index.html" {
+                config.service(web::resource("/").route(
+                    web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)),
+                ));
+            } else {
+                scope = scope.service(web::resource(path).route(
+                    web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)),
+                ));
+            }
+        }
+        config.service(scope);
+    } else {
+        config.service(web::resource("/").route(web::get().to(routes::running)));
+    }
+}
+
+#[cfg(not(feature = "mini-dashboard"))]
+pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) {
+    config.service(web::resource("/").route(web::get().to(routes::running)));
+}
+
+#[macro_export]
+macro_rules! create_app {
+    ($data:expr, $enable_frontend:expr) => {{
+        use actix_cors::Cors;
+        use actix_web::middleware::TrailingSlash;
+        use actix_web::App;
+        use actix_web::{middleware, web};
+        use meilisearch_http::routes::*;
+        use meilisearch_http::{configure_auth, configure_data, dashboard};
+
+        App::new()
+            .configure(|s| configure_data(s, $data.clone()))
+            .configure(|s| configure_auth(s, &$data))
+            .configure(document::services)
+            .configure(index::services)
+            .configure(search::services)
+            .configure(settings::services)
+            .configure(health::services)
+            .configure(stats::services)
+            .configure(key::services)
+            .configure(dump::services)
+            .configure(|s| dashboard(s, $enable_frontend))
+            .wrap(
+                Cors::default()
+                    .send_wildcard()
+                    .allowed_headers(vec!["content-type", "x-meili-api-key"])
+                    .allow_any_origin()
+                    .allow_any_method()
+                    .max_age(86_400), // 24h
+            )
+            .wrap(middleware::Logger::default())
+            .wrap(middleware::Compress::default())
+            .wrap(middleware::NormalizePath::new(
+                middleware::TrailingSlash::Trim,
+            ))
+    }};
+}

meilisearch-http/src/main.rs

@@ -0,0 +1,169 @@
+use std::env;
+
+use actix_web::HttpServer;
+use main_error::MainError;
+use meilisearch_http::{create_app, Data, Opt};
+use structopt::StructOpt;
+
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+use meilisearch_http::analytics;
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+use std::sync::Arc;
+
+#[cfg(target_os = "linux")]
+#[global_allocator]
+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
+
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+const SENTRY_DSN: &str = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337";
+
+#[actix_web::main]
+async fn main() -> Result<(), MainError> {
+    let opt = Opt::from_args();
+
+    let mut log_builder = env_logger::Builder::new();
+    log_builder.parse_filters(&opt.log_level);
+    if opt.log_level == "info" {
+        // if we are in info we only allow the warn log_level for milli
+        log_builder.filter_module("milli", log::LevelFilter::Warn);
+    }
+
+    match opt.env.as_ref() {
+        "production" => {
+            if opt.master_key.is_none() {
+                return Err(
+                    "In production mode, the environment variable MEILI_MASTER_KEY is mandatory"
+                        .into(),
+                );
+            }
+
+            #[cfg(all(not(debug_assertions), feature = "analytics"))]
+            if !opt.no_analytics {
+                let logger =
+                    sentry::integrations::log::SentryLogger::with_dest(log_builder.build());
+                log::set_boxed_logger(Box::new(logger))
+                    .map(|()| log::set_max_level(log::LevelFilter::Info))
+                    .unwrap();
+
+                let sentry = sentry::init(sentry::ClientOptions {
+                    release: sentry::release_name!(),
+                    dsn: Some(SENTRY_DSN.parse()?),
+                    before_send: Some(Arc::new(|event| {
+                        event
+                            .message
+                            .as_ref()
+                            .map(|msg| msg.to_lowercase().contains("no space left on device"))
+                            .unwrap_or(false)
+                            .then(|| event)
+                    })),
+                    ..Default::default()
+                });
+                // sentry must stay alive as long as possible
+                std::mem::forget(sentry);
+            } else {
+                log_builder.init();
+            }
+        }
+        "development" => {
+            log_builder.init();
+        }
+        _ => unreachable!(),
+    }
+
+    let data = Data::new(opt.clone())?;
+
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
+    if !opt.no_analytics {
+        let analytics_data = data.clone();
+        let analytics_opt = opt.clone();
+        tokio::task::spawn(analytics::analytics_sender(analytics_data, analytics_opt));
+    }
+
+    print_launch_resume(&opt, &data);
+
+    run_http(data, opt).await?;
+
+    Ok(())
+}
+
+async fn run_http(data: Data, opt: Opt) -> Result<(), Box<dyn std::error::Error>> {
+    let _enable_dashboard = &opt.env == "development";
+    let http_server = HttpServer::new(move || create_app!(data, _enable_dashboard))
+        // Disable signals allows the server to terminate immediately when a user enter CTRL-C
+        .disable_signals();
+
+    if let Some(config) = opt.get_ssl_config()? {
+        http_server
+            .bind_rustls(opt.http_addr, config)?
+            .run()
+            .await?;
+    } else {
+        http_server.bind(opt.http_addr)?.run().await?;
+    }
+    Ok(())
+}
+
+pub fn print_launch_resume(opt: &Opt, data: &Data) {
+    let commit_sha = match option_env!("COMMIT_SHA") {
+        Some("") | None => env!("VERGEN_SHA"),
+        Some(commit_sha) => commit_sha,
+    };
+    let commit_date = match option_env!("COMMIT_DATE") {
+        Some("") | None => env!("VERGEN_COMMIT_DATE"),
+        Some(commit_date) => commit_date,
+    };
+
+    let ascii_name = r#"
+888b     d888          d8b 888 d8b  .d8888b.                                    888
+8888b   d8888          Y8P 888 Y8P d88P  Y88b                                   888
+88888b.d88888              888     Y88b.                                        888
+888Y88888P888  .d88b.  888 888 888  "Y888b.    .d88b.   8888b.  888d888 .d8888b 88888b.
+888 Y888P 888 d8P  Y8b 888 888 888     "Y88b. d8P  Y8b     "88b 888P"  d88P"    888 "88b
+888  Y8P  888 88888888 888 888 888       "888 88888888 .d888888 888    888      888  888
+888   "   888 Y8b.     888 888 888 Y88b  d88P Y8b.     888  888 888    Y88b.    888  888
+888       888  "Y8888  888 888 888  "Y8888P"   "Y8888  "Y888888 888     "Y8888P 888  888
+"#;
+
+    eprintln!("{}", ascii_name);
+
+    eprintln!("Database path:\t\t{:?}", opt.db_path);
+    eprintln!("Server listening on:\t\"http://{}\"", opt.http_addr);
+    eprintln!("Environment:\t\t{:?}", opt.env);
+    eprintln!("Commit SHA:\t\t{:?}", commit_sha.to_string());
+    eprintln!("Commit date:\t\t{:?}", commit_date.to_string());
+    eprintln!(
+        "Package version:\t{:?}",
+        env!("CARGO_PKG_VERSION").to_string()
+    );
+
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
+    {
+        if opt.no_analytics {
+            eprintln!("Anonymous telemetry:\t\"Disabled\"");
+        } else {
+            eprintln!(
+                "
+Thank you for using MeiliSearch!
+
+We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics
+
+Anonymous telemetry:   \"Enabled\""
+            );
+        }
+    }
+
+    eprintln!();
+
+    if data.api_keys().master.is_some() {
+        eprintln!("A Master Key has been set. Requests to MeiliSearch won't be authorized unless you provide an authentication key.");
+    } else {
+        eprintln!("No master key found; The server will accept unidentified requests. \
+            If you need some protection in development mode, please export a key: export MEILI_MASTER_KEY=xxx");
+    }
+
+    eprintln!();
+    eprintln!("Documentation:\t\thttps://docs.meilisearch.com");
+    eprintln!("Source code:\t\thttps://github.com/meilisearch/meilisearch");
+    eprintln!("Contact:\t\thttps://docs.meilisearch.com/resources/contact.html or bonjour@meilisearch.com");
+    eprintln!();
+}

meilisearch-http/src/option.rs

@@ -0,0 +1,284 @@
+use std::io::{BufReader, Read};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::{error, fs};
+
+use byte_unit::Byte;
+use grenad::CompressionType;
+use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
+use rustls::{
+    AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
+    RootCertStore,
+};
+use structopt::StructOpt;
+
+#[derive(Debug, Clone, StructOpt)]
+pub struct IndexerOpts {
+    /// The amount of documents to skip before printing
+    /// a log regarding the indexing advancement.
+    #[structopt(long, default_value = "100000")] // 100k
+    pub log_every_n: usize,
+
+    /// Grenad max number of chunks in bytes.
+    #[structopt(long)]
+    pub max_nb_chunks: Option<usize>,
+
+    /// The maximum amount of memory to use for the Grenad buffer. It is recommended
+    /// to use something like 80%-90% of the available memory.
+    ///
+    /// It is automatically split by the number of jobs e.g. if you use 7 jobs
+    /// and 7 GB of max memory, each thread will use a maximum of 1 GB.
+    #[structopt(long, default_value = "7 GiB")]
+    pub max_memory: Byte,
+
+    /// Size of the linked hash map cache when indexing.
+    /// The bigger it is, the faster the indexing is but the more memory it takes.
+    #[structopt(long, default_value = "500")]
+    pub linked_hash_map_size: usize,
+
+    /// The name of the compression algorithm to use when compressing intermediate
+    /// Grenad chunks while indexing documents.
+    ///
+    /// Choosing a fast algorithm will make the indexing faster but may consume more memory.
+    #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
+    pub chunk_compression_type: CompressionType,
+
+    /// The level of compression of the chosen algorithm.
+    #[structopt(long, requires = "chunk-compression-type")]
+    pub chunk_compression_level: Option<u32>,
+
+    /// The number of bytes to remove from the begining of the chunks while reading/sorting
+    /// or merging them.
+    ///
+    /// File fusing must only be enable on file systems that support the `FALLOC_FL_COLLAPSE_RANGE`,
+    /// (i.e. ext4 and XFS). File fusing will only work if the `enable-chunk-fusing` is set.
+    #[structopt(long, default_value = "4 GiB")]
+    pub chunk_fusing_shrink_size: Byte,
+
+    /// Enable the chunk fusing or not, this reduces the amount of disk space used.
+    #[structopt(long)]
+    pub enable_chunk_fusing: bool,
+
+    /// Number of parallel jobs for indexing, defaults to # of CPUs.
+    #[structopt(long)]
+    pub indexing_jobs: Option<usize>,
+}
+
+impl Default for IndexerOpts {
+    fn default() -> Self {
+        Self {
+            log_every_n: 100_000,
+            max_nb_chunks: None,
+            max_memory: Byte::from_str("1GiB").unwrap(),
+            linked_hash_map_size: 500,
+            chunk_compression_type: CompressionType::None,
+            chunk_compression_level: None,
+            chunk_fusing_shrink_size: Byte::from_str("4GiB").unwrap(),
+            enable_chunk_fusing: false,
+            indexing_jobs: None,
+        }
+    }
+}
+
+const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
+
+#[derive(Debug, Clone, StructOpt)]
+pub struct Opt {
+    /// The destination where the database must be created.
+    #[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
+    pub db_path: PathBuf,
+
+    /// The address on which the http server will listen.
+    #[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")]
+    pub http_addr: String,
+
+    /// The master key allowing you to do everything on the server.
+    #[structopt(long, env = "MEILI_MASTER_KEY")]
+    pub master_key: Option<String>,
+
+    /// This environment variable must be set to `production` if you are running in production.
+    /// If the server is running in development mode more logs will be displayed,
+    /// and the master key can be avoided which implies that there is no security on the updates routes.
+    /// This is useful to debug when integrating the engine with another service.
+    #[structopt(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)]
+    pub env: String,
+
+    /// Do not send analytics to Meili.
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
+    #[structopt(long, env = "MEILI_NO_ANALYTICS")]
+    pub no_analytics: bool,
+
+    /// The maximum size, in bytes, of the main lmdb database directory
+    #[structopt(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")]
+    pub max_index_size: Byte,
+
+    /// The maximum size, in bytes, of the update lmdb database directory
+    #[structopt(long, env = "MEILI_MAX_UDB_SIZE", default_value = "100 GiB")]
+    pub max_udb_size: Byte,
+
+    /// The maximum size, in bytes, of accepted JSON payloads
+    #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")]
+    pub http_payload_size_limit: Byte,
+
+    /// Read server certificates from CERTFILE.
+    /// This should contain PEM-format certificates
+    /// in the right order (the first certificate should
+    /// certify KEYFILE, the last should be a root CA).
+    #[structopt(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))]
+    pub ssl_cert_path: Option<PathBuf>,
+
+    /// Read private key from KEYFILE.  This should be a RSA
+    /// private key or PKCS8-encoded private key, in PEM format.
+    #[structopt(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))]
+    pub ssl_key_path: Option<PathBuf>,
+
+    /// Enable client authentication, and accept certificates
+    /// signed by those roots provided in CERTFILE.
+    #[structopt(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))]
+    pub ssl_auth_path: Option<PathBuf>,
+
+    /// Read DER-encoded OCSP response from OCSPFILE and staple to certificate.
+    /// Optional
+    #[structopt(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))]
+    pub ssl_ocsp_path: Option<PathBuf>,
+
+    /// Send a fatal alert if the client does not complete client authentication.
+    #[structopt(long, env = "MEILI_SSL_REQUIRE_AUTH")]
+    pub ssl_require_auth: bool,
+
+    /// SSL support session resumption
+    #[structopt(long, env = "MEILI_SSL_RESUMPTION")]
+    pub ssl_resumption: bool,
+
+    /// SSL support tickets.
+    #[structopt(long, env = "MEILI_SSL_TICKETS")]
+    pub ssl_tickets: bool,
+
+    /// Defines the path of the snapshot file to import.
+    /// This option will, by default, stop the process if a database already exist or if no snapshot exists at
+    /// the given path. If this option is not specified no snapshot is imported.
+    #[structopt(long)]
+    pub import_snapshot: Option<PathBuf>,
+
+    /// The engine will ignore a missing snapshot and not return an error in such case.
+    #[structopt(long, requires = "import-snapshot")]
+    pub ignore_missing_snapshot: bool,
+
+    /// The engine will skip snapshot importation and not return an error in such case.
+    #[structopt(long, requires = "import-snapshot")]
+    pub ignore_snapshot_if_db_exists: bool,
+
+    /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap.
+    #[structopt(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")]
+    pub snapshot_dir: PathBuf,
+
+    /// Activate snapshot scheduling.
+    #[structopt(long, env = "MEILI_SCHEDULE_SNAPSHOT")]
+    pub schedule_snapshot: bool,
+
+    /// Defines time interval, in seconds, between each snapshot creation.
+    #[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h
+    pub snapshot_interval_sec: u64,
+
+    /// Folder where dumps are created when the dump route is called.
+    #[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")]
+    pub dumps_dir: PathBuf,
+
+    /// Import a dump from the specified path, must be a `.dump` file.
+    #[structopt(long, conflicts_with = "import-snapshot")]
+    pub import_dump: Option<PathBuf>,
+
+    /// Set the log level
+    #[structopt(long, env = "MEILI_LOG_LEVEL", default_value = "info")]
+    pub log_level: String,
+
+    #[structopt(skip)]
+    pub indexer_options: IndexerOpts,
+}
+
+impl Opt {
+    pub fn get_ssl_config(&self) -> Result<Option<rustls::ServerConfig>, Box<dyn error::Error>> {
+        if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) {
+            let client_auth = match &self.ssl_auth_path {
+                Some(auth_path) => {
+                    let roots = load_certs(auth_path.to_path_buf())?;
+                    let mut client_auth_roots = RootCertStore::empty();
+                    for root in roots {
+                        client_auth_roots.add(&root).unwrap();
+                    }
+                    if self.ssl_require_auth {
+                        AllowAnyAuthenticatedClient::new(client_auth_roots)
+                    } else {
+                        AllowAnyAnonymousOrAuthenticatedClient::new(client_auth_roots)
+                    }
+                }
+                None => NoClientAuth::new(),
+            };
+
+            let mut config = rustls::ServerConfig::new(client_auth);
+            config.key_log = Arc::new(rustls::KeyLogFile::new());
+
+            let certs = load_certs(cert_path.to_path_buf())?;
+            let privkey = load_private_key(key_path.to_path_buf())?;
+            let ocsp = load_ocsp(&self.ssl_ocsp_path)?;
+            config
+                .set_single_cert_with_ocsp_and_sct(certs, privkey, ocsp, vec![])
+                .map_err(|_| "bad certificates/private key")?;
+
+            if self.ssl_resumption {
+                config.set_persistence(rustls::ServerSessionMemoryCache::new(256));
+            }
+
+            if self.ssl_tickets {
+                config.ticketer = rustls::Ticketer::new();
+            }
+
+            Ok(Some(config))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+fn load_certs(filename: PathBuf) -> Result<Vec<rustls::Certificate>, Box<dyn error::Error>> {
+    let certfile = fs::File::open(filename).map_err(|_| "cannot open certificate file")?;
+    let mut reader = BufReader::new(certfile);
+    Ok(certs(&mut reader).map_err(|_| "cannot read certificate file")?)
+}
+
+fn load_private_key(filename: PathBuf) -> Result<rustls::PrivateKey, Box<dyn error::Error>> {
+    let rsa_keys = {
+        let keyfile =
+            fs::File::open(filename.clone()).map_err(|_| "cannot open private key file")?;
+        let mut reader = BufReader::new(keyfile);
+        rsa_private_keys(&mut reader).map_err(|_| "file contains invalid rsa private key")?
+    };
+
+    let pkcs8_keys = {
+        let keyfile = fs::File::open(filename).map_err(|_| "cannot open private key file")?;
+        let mut reader = BufReader::new(keyfile);
+        pkcs8_private_keys(&mut reader)
+            .map_err(|_| "file contains invalid pkcs8 private key (encrypted keys not supported)")?
+    };
+
+    // prefer to load pkcs8 keys
+    if !pkcs8_keys.is_empty() {
+        Ok(pkcs8_keys[0].clone())
+    } else {
+        assert!(!rsa_keys.is_empty());
+        Ok(rsa_keys[0].clone())
+    }
+}
+
+fn load_ocsp(filename: &Option<PathBuf>) -> Result<Vec<u8>, Box<dyn error::Error>> {
+    let mut ret = Vec::new();
+
+    if let Some(ref name) = filename {
+        fs::File::open(name)
+            .map_err(|_| "cannot open ocsp file")?
+            .read_to_end(&mut ret)
+            .map_err(|_| "cannot read oscp file")?;
+    }
+
+    Ok(ret)
+}

meilisearch-http/src/routes/document.rs

@@ -0,0 +1,213 @@
+use actix_web::{web, HttpResponse};
+use log::debug;
+use milli::update::{IndexDocumentsMethod, UpdateFormat};
+use serde::Deserialize;
+use serde_json::Value;
+
+use crate::error::ResponseError;
+use crate::extractors::authentication::{policies::*, GuardedData};
+use crate::extractors::payload::Payload;
+use crate::routes::IndexParam;
+use crate::Data;
+
+const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0;
+const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20;
+
+/*
+macro_rules! guard_content_type {
+    ($fn_name:ident, $guard_value:literal) => {
+        fn $fn_name(head: &actix_web::dev::RequestHead) -> bool {
+            if let Some(content_type) = head.headers.get("Content-Type") {
+                content_type
+                    .to_str()
+                    .map(|v| v.contains($guard_value))
+                    .unwrap_or(false)
+            } else {
+                false
+            }
+        }
+    };
+}
+
+guard_content_type!(guard_json, "application/json");
+*/
+
+fn guard_json(head: &actix_web::dev::RequestHead) -> bool {
+    if let Some(content_type) = head.headers.get("Content-Type") {
+        content_type
+            .to_str()
+            .map(|v| v.contains("application/json"))
+            .unwrap_or(false)
+    } else {
+        // if no content-type is specified we still accept the data as json!
+        true
+    }
+}
+
+#[derive(Deserialize)]
+struct DocumentParam {
+    index_uid: String,
+    document_id: String,
+}
+
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(
+        web::scope("/indexes/{index_uid}/documents")
+            .service(
+                web::resource("")
+                    .route(web::get().to(get_all_documents))
+                    .route(web::post().guard(guard_json).to(add_documents))
+                    .route(web::put().guard(guard_json).to(update_documents))
+                    .route(web::delete().to(clear_all_documents)),
+            )
+            // this route needs to be before the /documents/{document_id} to match properly
+            .service(web::resource("/delete-batch").route(web::post().to(delete_documents)))
+            .service(
+                web::resource("/{document_id}")
+                    .route(web::get().to(get_document))
+                    .route(web::delete().to(delete_document)),
+            ),
+    );
+}
+
+async fn get_document(
+    data: GuardedData<Public, Data>,
+    path: web::Path<DocumentParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let index = path.index_uid.clone();
+    let id = path.document_id.clone();
+    let document = data
+        .retrieve_document(index, id, None as Option<Vec<String>>)
+        .await?;
+    debug!("returns: {:?}", document);
+    Ok(HttpResponse::Ok().json(document))
+}
+
+async fn delete_document(
+    data: GuardedData<Private, Data>,
+    path: web::Path<DocumentParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let update_status = data
+        .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()])
+        .await?;
+    debug!("returns: {:?}", update_status);
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct BrowseQuery {
+    offset: Option<usize>,
+    limit: Option<usize>,
+    attributes_to_retrieve: Option<String>,
+}
+
+async fn get_all_documents(
+    data: GuardedData<Public, Data>,
+    path: web::Path<IndexParam>,
+    params: web::Query<BrowseQuery>,
+) -> Result<HttpResponse, ResponseError> {
+    debug!("called with params: {:?}", params);
+    let attributes_to_retrieve = params.attributes_to_retrieve.as_ref().and_then(|attrs| {
+        let mut names = Vec::new();
+        for name in attrs.split(',').map(String::from) {
+            if name == "*" {
+                return None;
+            }
+            names.push(name);
+        }
+        Some(names)
+    });
+
+    let documents = data
+        .retrieve_documents(
+            path.index_uid.clone(),
+            params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET),
+            params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT),
+            attributes_to_retrieve,
+        )
+        .await?;
+    debug!("returns: {:?}", documents);
+    Ok(HttpResponse::Ok().json(documents))
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct UpdateDocumentsQuery {
+    primary_key: Option<String>,
+}
+
+/// Route used when the payload type is "application/json"
+/// Used to add or replace documents
+async fn add_documents(
+    data: GuardedData<Private, Data>,
+    path: web::Path<IndexParam>,
+    params: web::Query<UpdateDocumentsQuery>,
+    body: Payload,
+) -> Result<HttpResponse, ResponseError> {
+    debug!("called with params: {:?}", params);
+    let update_status = data
+        .add_documents(
+            path.into_inner().index_uid,
+            IndexDocumentsMethod::ReplaceDocuments,
+            UpdateFormat::Json,
+            body,
+            params.primary_key.clone(),
+        )
+        .await?;
+
+    debug!("returns: {:?}", update_status);
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+}
+
+/// Route used when the payload type is "application/json"
+/// Used to add or replace documents
+async fn update_documents(
+    data: GuardedData<Private, Data>,
+    path: web::Path<IndexParam>,
+    params: web::Query<UpdateDocumentsQuery>,
+    body: Payload,
+) -> Result<HttpResponse, ResponseError> {
+    debug!("called with params: {:?}", params);
+    let update = data
+        .add_documents(
+            path.into_inner().index_uid,
+            IndexDocumentsMethod::UpdateDocuments,
+            UpdateFormat::Json,
+            body,
+            params.primary_key.clone(),
+        )
+        .await?;
+
+    debug!("returns: {:?}", update);
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() })))
+}
+
+async fn delete_documents(
+    data: GuardedData<Private, Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<Vec<Value>>,
+) -> Result<HttpResponse, ResponseError> {
+    debug!("called with params: {:?}", body);
+    let ids = body
+        .iter()
+        .map(|v| {
+            v.as_str()
+                .map(String::from)
+                .unwrap_or_else(|| v.to_string())
+        })
+        .collect();
+
+    let update_status = data.delete_documents(path.index_uid.clone(), ids).await?;
+    debug!("returns: {:?}", update_status);
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+}
+
+async fn clear_all_documents(
+    data: GuardedData<Private, Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let update_status = data.clear_documents(path.index_uid.clone()).await?;
+    debug!("returns: {:?}", update_status);
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+}

meilisearch-http/src/routes/dump.rs

@@ -0,0 +1,40 @@
+use actix_web::{web, HttpResponse};
+use log::debug;
+use serde::{Deserialize, Serialize};
+
+use crate::error::ResponseError;
+use crate::extractors::authentication::{policies::*, GuardedData};
+use crate::Data;
+
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::resource("/dumps").route(web::post().to(create_dump)))
+        .service(web::resource("/dumps/{dump_uid}/status").route(web::get().to(get_dump_status)));
+}
+
+async fn create_dump(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
+    let res = data.create_dump().await?;
+
+    debug!("returns: {:?}", res);
+    Ok(HttpResponse::Accepted().json(res))
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct DumpStatusResponse {
+    status: String,
+}
+
+#[derive(Deserialize)]
+struct DumpParam {
+    dump_uid: String,
+}
+
+async fn get_dump_status(
+    data: GuardedData<Private, Data>,
+    path: web::Path<DumpParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let res = data.dump_status(path.dump_uid.clone()).await?;
+
+    debug!("returns: {:?}", res);
+    Ok(HttpResponse::Ok().json(res))
+}

meilisearch-http/src/routes/health.rs

@@ -0,0 +1,11 @@
+use actix_web::{web, HttpResponse};
+
+use crate::error::ResponseError;
+
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::resource("/health").route(web::get().to(get_health)));
+}
+
+async fn get_health() -> Result<HttpResponse, ResponseError> {
+    Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" })))
+}

meilisearch-http/src/routes/index.rs

@@ -0,0 +1,133 @@
+use actix_web::{web, HttpResponse};
+use chrono::{DateTime, Utc};
+use log::debug;
+use serde::{Deserialize, Serialize};
+
+use super::{IndexParam, UpdateStatusResponse};
+use crate::error::ResponseError;
+use crate::extractors::authentication::{policies::*, GuardedData};
+use crate::Data;
+
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(
+        web::resource("indexes")
+            .route(web::get().to(list_indexes))
+            .route(web::post().to(create_index)),
+    )
+    .service(
+        web::resource("/indexes/{index_uid}")
+            .route(web::get().to(get_index))
+            .route(web::put().to(update_index))
+            .route(web::delete().to(delete_index)),
+    )
+    .service(
+        web::resource("/indexes/{index_uid}/updates").route(web::get().to(get_all_updates_status)),
+    )
+    .service(
+        web::resource("/indexes/{index_uid}/updates/{update_id}")
+            .route(web::get().to(get_update_status)),
+    );
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct IndexCreateRequest {
+    uid: String,
+    primary_key: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct UpdateIndexRequest {
+    uid: Option<String>,
+    primary_key: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UpdateIndexResponse {
+    name: String,
+    uid: String,
+    created_at: DateTime<Utc>,
+    updated_at: DateTime<Utc>,
+    primary_key: Option<String>,
+}
+
+async fn list_indexes(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
+    let indexes = data.list_indexes().await?;
+    debug!("returns: {:?}", indexes);
+    Ok(HttpResponse::Ok().json(indexes))
+}
+
+async fn create_index(
+    data: GuardedData<Private, Data>,
+    body: web::Json<IndexCreateRequest>,
+) -> Result<HttpResponse, ResponseError> {
+    let body = body.into_inner();
+    let meta = data.create_index(body.uid, body.primary_key).await?;
+    Ok(HttpResponse::Ok().json(meta))
+}
+
+async fn get_index(
+    data: GuardedData<Private, Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let meta = data.index(path.index_uid.clone()).await?;
+    debug!("returns: {:?}", meta);
+    Ok(HttpResponse::Ok().json(meta))
+}
+
+async fn update_index(
+    data: GuardedData<Private, Data>,
+    path: web::Path<IndexParam>,
+    body: web::Json<UpdateIndexRequest>,
+) -> Result<HttpResponse, ResponseError> {
+    debug!("called with params: {:?}", body);
+    let body = body.into_inner();
+    let meta = data
+        .update_index(path.into_inner().index_uid, body.primary_key, body.uid)
+        .await?;
+    debug!("returns: {:?}", meta);
+    Ok(HttpResponse::Ok().json(meta))
+}
+
+async fn delete_index(
+    data: GuardedData<Private, Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    data.delete_index(path.index_uid.clone()).await?;
+    Ok(HttpResponse::NoContent().finish())
+}
+
+#[derive(Deserialize)]
+struct UpdateParam {
+    index_uid: String,
+    update_id: u64,
+}
+
+async fn get_update_status(
+    data: GuardedData<Private, Data>,
+    path: web::Path<UpdateParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let params = path.into_inner();
+    let meta = data
+        .get_update_status(params.index_uid, params.update_id)
+        .await?;
+    let meta = UpdateStatusResponse::from(meta);
+    debug!("returns: {:?}", meta);
+    Ok(HttpResponse::Ok().json(meta))
+}
+
+async fn get_all_updates_status(
+    data: GuardedData<Private, Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let metas = data.get_updates_status(path.into_inner().index_uid).await?;
+    let metas = metas
+        .into_iter()
+        .map(UpdateStatusResponse::from)
+        .collect::<Vec<_>>();
+
+    debug!("returns: {:?}", metas);
+    Ok(HttpResponse::Ok().json(metas))
+}

meilisearch-http/src/routes/key.rs

@@ -0,0 +1,23 @@
+use actix_web::{web, HttpResponse};
+use serde::Serialize;
+
+use crate::extractors::authentication::{policies::*, GuardedData};
+use crate::Data;
+
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::resource("/keys").route(web::get().to(list)));
+}
+
+#[derive(Serialize)]
+struct KeysResponse {
+    private: Option<String>,
+    public: Option<String>,
+}
+
+async fn list(data: GuardedData<Admin, Data>) -> HttpResponse {
+    let api_keys = data.api_keys.clone();
+    HttpResponse::Ok().json(&KeysResponse {
+        private: api_keys.private,
+        public: api_keys.public,
+    })
+}

meilisearch-http/src/routes/mod.rs

@@ -0,0 +1,225 @@
+use std::time::Duration;
+
+use actix_web::HttpResponse;
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+
+use crate::error::ResponseError;
+use crate::index::{Settings, Unchecked};
+use crate::index_controller::{UpdateMeta, UpdateResult, UpdateStatus};
+
+pub mod document;
+pub mod dump;
+pub mod health;
+pub mod index;
+pub mod key;
+pub mod search;
+pub mod settings;
+pub mod stats;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[allow(clippy::large_enum_variant)]
+#[serde(tag = "name")]
+pub enum UpdateType {
+    ClearAll,
+    Customs,
+    DocumentsAddition {
+        #[serde(skip_serializing_if = "Option::is_none")]
+        number: Option<usize>,
+    },
+    DocumentsPartial {
+        #[serde(skip_serializing_if = "Option::is_none")]
+        number: Option<usize>,
+    },
+    DocumentsDeletion {
+        #[serde(skip_serializing_if = "Option::is_none")]
+        number: Option<usize>,
+    },
+    Settings {
+        settings: Settings<Unchecked>,
+    },
+}
+
+impl From<&UpdateStatus> for UpdateType {
+    fn from(other: &UpdateStatus) -> Self {
+        use milli::update::IndexDocumentsMethod::*;
+
+        match other.meta() {
+            UpdateMeta::DocumentsAddition { method, .. } => {
+                let number = match other {
+                    UpdateStatus::Processed(processed) => match processed.success {
+                        UpdateResult::DocumentsAddition(ref addition) => {
+                            Some(addition.nb_documents)
+                        }
+                        _ => None,
+                    },
+                    _ => None,
+                };
+
+                match method {
+                    ReplaceDocuments => UpdateType::DocumentsAddition { number },
+                    UpdateDocuments => UpdateType::DocumentsPartial { number },
+                    _ => unreachable!(),
+                }
+            }
+            UpdateMeta::ClearDocuments => UpdateType::ClearAll,
+            UpdateMeta::DeleteDocuments { ids } => UpdateType::DocumentsDeletion {
+                number: Some(ids.len()),
+            },
+            UpdateMeta::Settings(settings) => UpdateType::Settings {
+                settings: settings.clone(),
+            },
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ProcessedUpdateResult {
+    pub update_id: u64,
+    #[serde(rename = "type")]
+    pub update_type: UpdateType,
+    pub duration: f64, // in seconds
+    pub enqueued_at: DateTime<Utc>,
+    pub processed_at: DateTime<Utc>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FailedUpdateResult {
+    pub update_id: u64,
+    #[serde(rename = "type")]
+    pub update_type: UpdateType,
+    #[serde(flatten)]
+    pub response: ResponseError,
+    pub duration: f64, // in seconds
+    pub enqueued_at: DateTime<Utc>,
+    pub processed_at: DateTime<Utc>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct EnqueuedUpdateResult {
+    pub update_id: u64,
+    #[serde(rename = "type")]
+    pub update_type: UpdateType,
+    pub enqueued_at: DateTime<Utc>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub started_processing_at: Option<DateTime<Utc>>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase", tag = "status")]
+pub enum UpdateStatusResponse {
+    Enqueued {
+        #[serde(flatten)]
+        content: EnqueuedUpdateResult,
+    },
+    Processing {
+        #[serde(flatten)]
+        content: EnqueuedUpdateResult,
+    },
+    Failed {
+        #[serde(flatten)]
+        content: FailedUpdateResult,
+    },
+    Processed {
+        #[serde(flatten)]
+        content: ProcessedUpdateResult,
+    },
+}
+
+impl From<UpdateStatus> for UpdateStatusResponse {
+    fn from(other: UpdateStatus) -> Self {
+        let update_type = UpdateType::from(&other);
+
+        match other {
+            UpdateStatus::Processing(processing) => {
+                let content = EnqueuedUpdateResult {
+                    update_id: processing.id(),
+                    update_type,
+                    enqueued_at: processing.from.enqueued_at,
+                    started_processing_at: Some(processing.started_processing_at),
+                };
+                UpdateStatusResponse::Processing { content }
+            }
+            UpdateStatus::Enqueued(enqueued) => {
+                let content = EnqueuedUpdateResult {
+                    update_id: enqueued.id(),
+                    update_type,
+                    enqueued_at: enqueued.enqueued_at,
+                    started_processing_at: None,
+                };
+                UpdateStatusResponse::Enqueued { content }
+            }
+            UpdateStatus::Processed(processed) => {
+                let duration = processed
+                    .processed_at
+                    .signed_duration_since(processed.from.started_processing_at)
+                    .num_milliseconds();
+
+                // necessary since chrono::duration don't expose a f64 secs method.
+                let duration = Duration::from_millis(duration as u64).as_secs_f64();
+
+                let content = ProcessedUpdateResult {
+                    update_id: processed.id(),
+                    update_type,
+                    duration,
+                    enqueued_at: processed.from.from.enqueued_at,
+                    processed_at: processed.processed_at,
+                };
+                UpdateStatusResponse::Processed { content }
+            }
+            UpdateStatus::Aborted(_) => unreachable!(),
+            UpdateStatus::Failed(failed) => {
+                let duration = failed
+                    .failed_at
+                    .signed_duration_since(failed.from.started_processing_at)
+                    .num_milliseconds();
+
+                // necessary since chrono::duration don't expose a f64 secs method.
+                let duration = Duration::from_millis(duration as u64).as_secs_f64();
+
+                let update_id = failed.id();
+                let response = failed.error;
+
+                let content = FailedUpdateResult {
+                    update_id,
+                    update_type,
+                    response,
+                    duration,
+                    enqueued_at: failed.from.from.enqueued_at,
+                    processed_at: failed.failed_at,
+                };
+                UpdateStatusResponse::Failed { content }
+            }
+        }
+    }
+}
+
+#[derive(Deserialize)]
+pub struct IndexParam {
+    index_uid: String,
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct IndexUpdateResponse {
+    pub update_id: u64,
+}
+
+impl IndexUpdateResponse {
+    pub fn with_id(update_id: u64) -> Self {
+        Self { update_id }
+    }
+}
+
+/// Always return a 200 with:
+/// ```json
+/// {
+///     "status": "Meilisearch is running"
+/// }
+/// ```
+pub async fn running() -> HttpResponse {
+    HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" }))
+}

meilisearch-http/src/routes/search.rs

@@ -0,0 +1,103 @@
+use std::collections::{BTreeSet, HashSet};
+
+use actix_web::{web, HttpResponse};
+use log::debug;
+use serde::Deserialize;
+use serde_json::Value;
+
+use crate::error::ResponseError;
+use crate::extractors::authentication::{policies::*, GuardedData};
+use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT};
+use crate::routes::IndexParam;
+use crate::Data;
+
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(
+        web::resource("/indexes/{index_uid}/search")
+            .route(web::get().to(search_with_url_query))
+            .route(web::post().to(search_with_post)),
+    );
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct SearchQueryGet {
+    q: Option<String>,
+    offset: Option<usize>,
+    limit: Option<usize>,
+    attributes_to_retrieve: Option<String>,
+    attributes_to_crop: Option<String>,
+    #[serde(default = "default_crop_length")]
+    crop_length: usize,
+    attributes_to_highlight: Option<String>,
+    filter: Option<String>,
+    #[serde(default = "Default::default")]
+    matches: bool,
+    facets_distribution: Option<String>,
+}
+
+impl From<SearchQueryGet> for SearchQuery {
+    fn from(other: SearchQueryGet) -> Self {
+        let attributes_to_retrieve = other
+            .attributes_to_retrieve
+            .map(|attrs| attrs.split(',').map(String::from).collect::<BTreeSet<_>>());
+
+        let attributes_to_crop = other
+            .attributes_to_crop
+            .map(|attrs| attrs.split(',').map(String::from).collect::<Vec<_>>());
+
+        let attributes_to_highlight = other
+            .attributes_to_highlight
+            .map(|attrs| attrs.split(',').map(String::from).collect::<HashSet<_>>());
+
+        let facets_distribution = other
+            .facets_distribution
+            .map(|attrs| attrs.split(',').map(String::from).collect::<Vec<_>>());
+
+        let filter = match other.filter {
+            Some(f) => match serde_json::from_str(&f) {
+                Ok(v) => Some(v),
+                _ => Some(Value::String(f)),
+            },
+            None => None,
+        };
+
+        Self {
+            q: other.q,
+            offset: other.offset,
+            limit: other.limit.unwrap_or(DEFAULT_SEARCH_LIMIT),
+            attributes_to_retrieve,
+            attributes_to_crop,
+            crop_length: other.crop_length,
+            attributes_to_highlight,
+            filter,
+            matches: other.matches,
+            facets_distribution,
+        }
+    }
+}
+
+async fn search_with_url_query(
+    data: GuardedData<Admin, Data>,
+    path: web::Path<IndexParam>,
+    params: web::Query<SearchQueryGet>,
+) -> Result<HttpResponse, ResponseError> {
+    debug!("called with params: {:?}", params);
+    let query = params.into_inner().into();
+    let search_result = data.search(path.into_inner().index_uid, query).await?;
+    debug!("returns: {:?}", search_result);
+    Ok(HttpResponse::Ok().json(search_result))
+}
+
+async fn search_with_post(
+    data: GuardedData<Public, Data>,
+    path: web::Path<IndexParam>,
+    params: web::Json<SearchQuery>,
+) -> Result<HttpResponse, ResponseError> {
+    debug!("search called with params: {:?}", params);
+    let search_result = data
+        .search(path.into_inner().index_uid, params.into_inner())
+        .await?;
+    debug!("returns: {:?}", search_result);
+    Ok(HttpResponse::Ok().json(search_result))
+}

meilisearch-http/src/routes/settings.rs

@@ -0,0 +1,177 @@
+use actix_web::{web, HttpResponse};
+use log::debug;
+
+use crate::extractors::authentication::{policies::*, GuardedData};
+use crate::index::Settings;
+use crate::Data;
+use crate::{error::ResponseError, index::Unchecked};
+
+#[macro_export]
+macro_rules! make_setting_route {
+    ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => {
+        mod $attr {
+            use log::debug;
+            use actix_web::{web, HttpResponse, Resource};
+
+            use crate::data;
+            use crate::error::ResponseError;
+            use crate::index::Settings;
+            use crate::extractors::authentication::{GuardedData, policies::*};
+
+            async fn delete(
+                data: GuardedData<Private, data::Data>,
+                index_uid: web::Path<String>,
+            ) -> Result<HttpResponse, ResponseError> {
+                use crate::index::Settings;
+                let settings = Settings {
+                    $attr: Some(None),
+                    ..Default::default()
+                };
+                let update_status = data.update_settings(index_uid.into_inner(), settings, false).await?;
+                debug!("returns: {:?}", update_status);
+                Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+            }
+
+            async fn update(
+                data: GuardedData<Private, data::Data>,
+                index_uid: actix_web::web::Path<String>,
+                body: actix_web::web::Json<Option<$type>>,
+            ) -> std::result::Result<HttpResponse, ResponseError> {
+                let settings = Settings {
+                    $attr: Some(body.into_inner()),
+                    ..Default::default()
+                };
+
+                let update_status = data.update_settings(index_uid.into_inner(), settings, true).await?;
+                debug!("returns: {:?}", update_status);
+                Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+            }
+
+            async fn get(
+                data: GuardedData<Private, data::Data>,
+                index_uid: actix_web::web::Path<String>,
+            ) -> std::result::Result<HttpResponse, ResponseError> {
+                let settings = data.settings(index_uid.into_inner()).await?;
+                debug!("returns: {:?}", settings);
+                let mut json = serde_json::json!(&settings);
+                let val = json[$camelcase_attr].take();
+                Ok(HttpResponse::Ok().json(val))
+            }
+
+            pub fn resources() -> Resource {
+                Resource::new($route)
+                    .route(web::get().to(get))
+                    .route(web::post().to(update))
+                    .route(web::delete().to(delete))
+            }
+        }
+    };
+}
+
+make_setting_route!(
+    "/indexes/{index_uid}/settings/filterable-attributes",
+    std::collections::HashSet<String>,
+    filterable_attributes,
+    "filterableAttributes"
+);
+
+make_setting_route!(
+    "/indexes/{index_uid}/settings/displayed-attributes",
+    Vec<String>,
+    displayed_attributes,
+    "displayedAttributes"
+);
+
+make_setting_route!(
+    "/indexes/{index_uid}/settings/searchable-attributes",
+    Vec<String>,
+    searchable_attributes,
+    "searchableAttributes"
+);
+
+make_setting_route!(
+    "/indexes/{index_uid}/settings/stop-words",
+    std::collections::BTreeSet<String>,
+    stop_words,
+    "stopWords"
+);
+
+make_setting_route!(
+    "/indexes/{index_uid}/settings/synonyms",
+    std::collections::BTreeMap<String, Vec<String>>,
+    synonyms,
+    "synonyms"
+);
+
+make_setting_route!(
+    "/indexes/{index_uid}/settings/distinct-attribute",
+    String,
+    distinct_attribute,
+    "distinctAttribute"
+);
+
+make_setting_route!(
+    "/indexes/{index_uid}/settings/ranking-rules",
+    Vec<String>,
+    ranking_rules,
+    "rankingRules"
+);
+
+macro_rules! create_services {
+    ($($mod:ident),*) => {
+        pub fn services(cfg: &mut web::ServiceConfig) {
+            cfg
+                .service(web::resource("/indexes/{index_uid}/settings")
+                    .route(web::post().to(update_all))
+                    .route(web::get().to(get_all))
+                    .route(web::delete().to(delete_all)))
+                $(.service($mod::resources()))*;
+        }
+    };
+}
+
+create_services!(
+    filterable_attributes,
+    displayed_attributes,
+    searchable_attributes,
+    distinct_attribute,
+    stop_words,
+    synonyms,
+    ranking_rules
+);
+
+async fn update_all(
+    data: GuardedData<Private, Data>,
+    index_uid: web::Path<String>,
+    body: web::Json<Settings<Unchecked>>,
+) -> Result<HttpResponse, ResponseError> {
+    let settings = body.into_inner().check();
+    let update_result = data
+        .update_settings(index_uid.into_inner(), settings, true)
+        .await?;
+    let json = serde_json::json!({ "updateId": update_result.id() });
+    debug!("returns: {:?}", json);
+    Ok(HttpResponse::Accepted().json(json))
+}
+
+async fn get_all(
+    data: GuardedData<Private, Data>,
+    index_uid: web::Path<String>,
+) -> Result<HttpResponse, ResponseError> {
+    let settings = data.settings(index_uid.into_inner()).await?;
+    debug!("returns: {:?}", settings);
+    Ok(HttpResponse::Ok().json(settings))
+}
+
+async fn delete_all(
+    data: GuardedData<Private, Data>,
+    index_uid: web::Path<String>,
+) -> Result<HttpResponse, ResponseError> {
+    let settings = Settings::cleared();
+    let update_result = data
+        .update_settings(index_uid.into_inner(), settings, false)
+        .await?;
+    let json = serde_json::json!({ "updateId": update_result.id() });
+    debug!("returns: {:?}", json);
+    Ok(HttpResponse::Accepted().json(json))
+}

meilisearch-http/src/routes/stats.rs

@@ -0,0 +1,56 @@
+use actix_web::{web, HttpResponse};
+use log::debug;
+use serde::Serialize;
+
+use crate::error::ResponseError;
+use crate::extractors::authentication::{policies::*, GuardedData};
+use crate::routes::IndexParam;
+use crate::Data;
+
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::resource("/indexes/{index_uid}/stats").route(web::get().to(get_index_stats)))
+        .service(web::resource("/stats").route(web::get().to(get_stats)))
+        .service(web::resource("/version").route(web::get().to(get_version)));
+}
+
+async fn get_index_stats(
+    data: GuardedData<Private, Data>,
+    path: web::Path<IndexParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let response = data.get_index_stats(path.index_uid.clone()).await?;
+
+    debug!("returns: {:?}", response);
+    Ok(HttpResponse::Ok().json(response))
+}
+
+async fn get_stats(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
+    let response = data.get_all_stats().await?;
+
+    debug!("returns: {:?}", response);
+    Ok(HttpResponse::Ok().json(response))
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct VersionResponse {
+    commit_sha: String,
+    commit_date: String,
+    pkg_version: String,
+}
+
+async fn get_version(_data: GuardedData<Private, Data>) -> HttpResponse {
+    let commit_sha = match option_env!("COMMIT_SHA") {
+        Some("") | None => env!("VERGEN_SHA"),
+        Some(commit_sha) => commit_sha,
+    };
+    let commit_date = match option_env!("COMMIT_DATE") {
+        Some("") | None => env!("VERGEN_COMMIT_DATE"),
+        Some(commit_date) => commit_date,
+    };
+
+    HttpResponse::Ok().json(VersionResponse {
+        commit_sha: commit_sha.to_string(),
+        commit_date: commit_date.to_string(),
+        pkg_version: env!("CARGO_PKG_VERSION").to_string(),
+    })
+}

meilisearch-http/tests/assets/dumps/v1/metadata.json

@@ -0,0 +1,12 @@
+{
+    "indices": [{
+            "uid": "test",
+            "primaryKey": "id"
+        }, {
+            "uid": "test2",
+            "primaryKey": "test2_id"
+        }
+    ],
+    "dbVersion": "0.13.0",
+    "dumpVersion": "1"
+}

meilisearch-http/tests/assets/dumps/v1/test/documents.jsonl

@@ -0,0 +1,77 @@
+{"id":0,"isActive":false,"balance":"$2,668.55","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Lucas Hess","gender":"male","email":"lucashess@chorizon.com","phone":"+1 (998) 478-2597","address":"412 Losee Terrace, Blairstown, Georgia, 2825","about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n","registered":"2016-06-21T09:30:25 -02:00","latitude":-44.174957,"longitude":-145.725388,"tags":["bug","bug"]}
+{"id":1,"isActive":true,"balance":"$1,706.13","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Cherry Orr","gender":"female","email":"cherryorr@chorizon.com","phone":"+1 (995) 479-3174","address":"442 Beverly Road, Ventress, New Mexico, 3361","about":"Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n","registered":"2020-03-18T11:12:21 -01:00","latitude":-24.356932,"longitude":27.184808,"tags":["new issue","bug"]}
+{"id":2,"isActive":true,"balance":"$2,467.47","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Patricia Goff","gender":"female","email":"patriciagoff@chorizon.com","phone":"+1 (864) 463-2277","address":"866 Hornell Loop, Cresaptown, Ohio, 1700","about":"Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n","registered":"2014-10-28T12:59:30 -01:00","latitude":-64.008555,"longitude":11.867098,"tags":["good first issue"]}
+{"id":3,"isActive":true,"balance":"$3,344.40","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Adeline Flynn","gender":"female","email":"adelineflynn@chorizon.com","phone":"+1 (994) 600-2840","address":"428 Paerdegat Avenue, Hollymead, Pennsylvania, 948","about":"Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n","registered":"2014-03-27T06:24:45 -01:00","latitude":-74.485173,"longitude":-11.059859,"tags":["bug","good first issue","wontfix","new issue"]}
+{"id":4,"isActive":false,"balance":"$2,575.78","picture":"http://placehold.it/32x32","age":39,"color":"Green","name":"Mariana Pacheco","gender":"female","email":"marianapacheco@chorizon.com","phone":"+1 (820) 414-2223","address":"664 Rapelye Street, Faywood, California, 7320","about":"Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n","registered":"2015-09-02T03:23:35 -02:00","latitude":75.763501,"longitude":-78.777124,"tags":["new issue"]}
+{"id":5,"isActive":true,"balance":"$3,793.09","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Warren Watson","gender":"male","email":"warrenwatson@chorizon.com","phone":"+1 (807) 583-2427","address":"671 Prince Street, Faxon, Connecticut, 4275","about":"Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n","registered":"2017-06-04T06:02:17 -02:00","latitude":29.979223,"longitude":25.358943,"tags":["wontfix","wontfix","wontfix"]}
+{"id":6,"isActive":true,"balance":"$2,919.70","picture":"http://placehold.it/32x32","age":20,"color":"blue","name":"Shelia Berry","gender":"female","email":"sheliaberry@chorizon.com","phone":"+1 (853) 511-2651","address":"437 Forrest Street, Coventry, Illinois, 2056","about":"Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n","registered":"2018-07-11T02:45:01 -02:00","latitude":54.815991,"longitude":-118.690609,"tags":["good first issue","bug","wontfix","new issue"]}
+{"id":7,"isActive":true,"balance":"$1,349.50","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Chrystal Boyd","gender":"female","email":"chrystalboyd@chorizon.com","phone":"+1 (936) 563-2802","address":"670 Croton Loop, Sussex, Florida, 4692","about":"Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n","registered":"2016-11-01T07:36:04 -01:00","latitude":-24.711933,"longitude":147.246705,"tags":[]}
+{"id":8,"isActive":false,"balance":"$3,999.56","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Martin Porter","gender":"male","email":"martinporter@chorizon.com","phone":"+1 (895) 580-2304","address":"577 Regent Place, Aguila, Guam, 6554","about":"Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n","registered":"2014-09-20T02:08:30 -02:00","latitude":-88.344273,"longitude":37.964466,"tags":[]}
+{"id":9,"isActive":true,"balance":"$3,729.71","picture":"http://placehold.it/32x32","age":26,"color":"blue","name":"Kelli Mendez","gender":"female","email":"kellimendez@chorizon.com","phone":"+1 (936) 401-2236","address":"242 Caton Place, Grazierville, Alabama, 3968","about":"Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n","registered":"2018-05-04T10:35:30 -02:00","latitude":49.37551,"longitude":41.872323,"tags":["new issue","new issue"]}
+{"id":10,"isActive":false,"balance":"$1,127.47","picture":"http://placehold.it/32x32","age":27,"color":"blue","name":"Maddox Johns","gender":"male","email":"maddoxjohns@chorizon.com","phone":"+1 (892) 470-2357","address":"756 Beard Street, Avalon, Louisiana, 114","about":"Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n","registered":"2016-04-22T06:41:25 -02:00","latitude":66.640229,"longitude":-17.222666,"tags":["new issue","good first issue","good first issue","new issue"]}
+{"id":11,"isActive":true,"balance":"$1,351.43","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Evans Wagner","gender":"male","email":"evanswagner@chorizon.com","phone":"+1 (889) 496-2332","address":"118 Monaco Place, Lutsen, Delaware, 6209","about":"Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n","registered":"2016-10-27T01:26:31 -02:00","latitude":-77.673222,"longitude":-142.657214,"tags":["good first issue","good first issue"]}
+{"id":12,"isActive":false,"balance":"$3,394.96","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Aida Kirby","gender":"female","email":"aidakirby@chorizon.com","phone":"+1 (942) 532-2325","address":"797 Engert Avenue, Wilsonia, Idaho, 6532","about":"Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n","registered":"2018-06-18T04:39:57 -02:00","latitude":-58.062041,"longitude":34.999254,"tags":["new issue","wontfix","bug","new issue"]}
+{"id":13,"isActive":true,"balance":"$2,812.62","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Nelda Burris","gender":"female","email":"neldaburris@chorizon.com","phone":"+1 (813) 600-2576","address":"160 Opal Court, Fowlerville, Tennessee, 2170","about":"Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n","registered":"2015-08-15T12:39:53 -02:00","latitude":66.6871,"longitude":179.549488,"tags":["wontfix"]}
+{"id":14,"isActive":true,"balance":"$1,718.33","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Jennifer Hart","gender":"female","email":"jenniferhart@chorizon.com","phone":"+1 (850) 537-2513","address":"124 Veranda Place, Nash, Utah, 985","about":"Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n","registered":"2016-09-04T11:46:59 -02:00","latitude":-66.827751,"longitude":99.220079,"tags":["wontfix","bug","new issue","new issue"]}
+{"id":15,"isActive":false,"balance":"$2,698.16","picture":"http://placehold.it/32x32","age":28,"color":"blue","name":"Aurelia Contreras","gender":"female","email":"aureliacontreras@chorizon.com","phone":"+1 (932) 442-3103","address":"655 Dwight Street, Grapeview, Palau, 8356","about":"Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n","registered":"2014-09-11T10:43:15 -02:00","latitude":-71.328973,"longitude":133.404895,"tags":["wontfix","bug","good first issue"]}
+{"id":16,"isActive":true,"balance":"$3,303.25","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Estella Bass","gender":"female","email":"estellabass@chorizon.com","phone":"+1 (825) 436-2909","address":"435 Rockwell Place, Garberville, Wisconsin, 2230","about":"Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n","registered":"2017-11-23T09:32:09 -01:00","latitude":81.17014,"longitude":-145.262693,"tags":["new issue"]}
+{"id":17,"isActive":false,"balance":"$3,579.20","picture":"http://placehold.it/32x32","age":25,"color":"brown","name":"Ortega Brennan","gender":"male","email":"ortegabrennan@chorizon.com","phone":"+1 (906) 526-2287","address":"440 Berry Street, Rivera, Maine, 1849","about":"Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n","registered":"2016-03-31T02:17:13 -02:00","latitude":-68.407524,"longitude":-113.642067,"tags":["new issue","wontfix"]}
+{"id":18,"isActive":false,"balance":"$1,484.92","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Leonard Tillman","gender":"male","email":"leonardtillman@chorizon.com","phone":"+1 (864) 541-3456","address":"985 Provost Street, Charco, New Hampshire, 8632","about":"Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n","registered":"2018-05-06T08:21:27 -02:00","latitude":-8.581801,"longitude":-61.910062,"tags":["wontfix","new issue","bug","bug"]}
+{"id":19,"isActive":true,"balance":"$3,572.55","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Dale Payne","gender":"male","email":"dalepayne@chorizon.com","phone":"+1 (814) 469-3499","address":"536 Dare Court, Ironton, Arkansas, 8605","about":"Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n","registered":"2019-10-11T01:01:33 -02:00","latitude":-18.280968,"longitude":-126.091797,"tags":["bug","wontfix","wontfix","wontfix"]}
+{"id":20,"isActive":true,"balance":"$1,986.48","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Florence Long","gender":"female","email":"florencelong@chorizon.com","phone":"+1 (972) 557-3858","address":"519 Hendrickson Street, Templeton, Hawaii, 2389","about":"Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n","registered":"2016-05-02T09:18:59 -02:00","latitude":-27.110866,"longitude":-45.09445,"tags":[]}
+{"id":21,"isActive":true,"balance":"$1,440.09","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Levy Whitley","gender":"male","email":"levywhitley@chorizon.com","phone":"+1 (911) 458-2411","address":"187 Thomas Street, Hachita, North Carolina, 2989","about":"Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n","registered":"2014-04-30T07:31:38 -02:00","latitude":-6.537315,"longitude":171.813536,"tags":["bug"]}
+{"id":22,"isActive":false,"balance":"$2,938.57","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Bernard Mcfarland","gender":"male","email":"bernardmcfarland@chorizon.com","phone":"+1 (979) 442-3386","address":"409 Hall Street, Keyport, Federated States Of Micronesia, 7011","about":"Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n","registered":"2017-08-10T10:07:59 -02:00","latitude":63.766795,"longitude":68.177069,"tags":[]}
+{"id":23,"isActive":true,"balance":"$1,678.49","picture":"http://placehold.it/32x32","age":31,"color":"brown","name":"Blanca Mcclain","gender":"female","email":"blancamcclain@chorizon.com","phone":"+1 (976) 439-2772","address":"176 Crooke Avenue, Valle, Virginia, 5373","about":"Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n","registered":"2015-10-12T11:57:28 -02:00","latitude":-8.944564,"longitude":-150.711709,"tags":["bug","wontfix","good first issue"]}
+{"id":24,"isActive":true,"balance":"$2,276.87","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Espinoza Ford","gender":"male","email":"espinozaford@chorizon.com","phone":"+1 (945) 429-3975","address":"137 Bowery Street, Itmann, District Of Columbia, 1864","about":"Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n","registered":"2014-03-26T02:16:08 -01:00","latitude":-37.137666,"longitude":-51.811757,"tags":["wontfix","bug"]}
+{"id":25,"isActive":true,"balance":"$3,973.43","picture":"http://placehold.it/32x32","age":29,"color":"Green","name":"Sykes Conley","gender":"male","email":"sykesconley@chorizon.com","phone":"+1 (851) 401-3916","address":"345 Grand Street, Woodlands, Missouri, 4461","about":"Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n","registered":"2015-09-12T06:03:56 -02:00","latitude":67.282955,"longitude":-64.341323,"tags":["wontfix"]}
+{"id":26,"isActive":false,"balance":"$1,431.50","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Barlow Duran","gender":"male","email":"barlowduran@chorizon.com","phone":"+1 (995) 436-2562","address":"481 Everett Avenue, Allison, Nebraska, 3065","about":"Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n","registered":"2017-06-29T04:28:43 -02:00","latitude":-38.70606,"longitude":55.02816,"tags":["new issue"]}
+{"id":27,"isActive":true,"balance":"$3,478.27","picture":"http://placehold.it/32x32","age":31,"color":"blue","name":"Schwartz Morgan","gender":"male","email":"schwartzmorgan@chorizon.com","phone":"+1 (861) 507-2067","address":"451 Lincoln Road, Fairlee, Washington, 2717","about":"Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n","registered":"2016-05-10T08:34:54 -02:00","latitude":-75.886403,"longitude":93.044471,"tags":["bug","bug","wontfix","wontfix"]}
+{"id":28,"isActive":true,"balance":"$2,825.59","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Kristy Leon","gender":"female","email":"kristyleon@chorizon.com","phone":"+1 (948) 465-2563","address":"594 Macon Street, Floris, South Dakota, 3565","about":"Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n","registered":"2014-12-14T04:10:29 -01:00","latitude":-50.01615,"longitude":-68.908804,"tags":["wontfix","good first issue"]}
+{"id":29,"isActive":false,"balance":"$3,028.03","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Ashley Pittman","gender":"male","email":"ashleypittman@chorizon.com","phone":"+1 (928) 507-3523","address":"646 Adelphi Street, Clara, Colorado, 6056","about":"Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n","registered":"2016-01-07T10:40:48 -01:00","latitude":-58.766037,"longitude":-124.828485,"tags":["wontfix"]}
+{"id":30,"isActive":true,"balance":"$2,021.11","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Stacy Espinoza","gender":"female","email":"stacyespinoza@chorizon.com","phone":"+1 (999) 487-3253","address":"931 Alabama Avenue, Bangor, Alaska, 8215","about":"Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n","registered":"2014-07-16T06:15:53 -02:00","latitude":41.560197,"longitude":177.697,"tags":["new issue","new issue","bug"]}
+{"id":31,"isActive":false,"balance":"$3,609.82","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Vilma Garza","gender":"female","email":"vilmagarza@chorizon.com","phone":"+1 (944) 585-2021","address":"565 Tech Place, Sedley, Puerto Rico, 858","about":"Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n","registered":"2017-06-30T07:43:52 -02:00","latitude":-12.574889,"longitude":-54.771186,"tags":["new issue","wontfix","wontfix"]}
+{"id":32,"isActive":false,"balance":"$2,882.34","picture":"http://placehold.it/32x32","age":38,"color":"brown","name":"June Dunlap","gender":"female","email":"junedunlap@chorizon.com","phone":"+1 (997) 504-2937","address":"353 Cozine Avenue, Goodville, Indiana, 1438","about":"Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n","registered":"2016-08-23T08:54:11 -02:00","latitude":-27.883363,"longitude":-163.919683,"tags":["new issue","new issue","bug","wontfix"]}
+{"id":33,"isActive":true,"balance":"$3,556.54","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Cecilia Greer","gender":"female","email":"ceciliagreer@chorizon.com","phone":"+1 (977) 573-3498","address":"696 Withers Street, Lydia, Oklahoma, 3220","about":"Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n","registered":"2017-01-13T11:30:12 -01:00","latitude":60.467215,"longitude":84.684575,"tags":["wontfix","good first issue","good first issue","wontfix"]}
+{"id":34,"isActive":true,"balance":"$1,413.35","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Mckay Schroeder","gender":"male","email":"mckayschroeder@chorizon.com","phone":"+1 (816) 480-3657","address":"958 Miami Court, Rehrersburg, Northern Mariana Islands, 567","about":"Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n","registered":"2016-02-08T04:50:15 -01:00","latitude":-72.413287,"longitude":-159.254371,"tags":["good first issue"]}
+{"id":35,"isActive":true,"balance":"$2,306.53","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Sawyer Mccormick","gender":"male","email":"sawyermccormick@chorizon.com","phone":"+1 (829) 569-3012","address":"749 Apollo Street, Eastvale, Texas, 7373","about":"Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n","registered":"2019-11-30T11:53:23 -01:00","latitude":-48.978194,"longitude":110.950191,"tags":["good first issue","new issue","new issue","bug"]}
+{"id":36,"isActive":false,"balance":"$1,844.54","picture":"http://placehold.it/32x32","age":37,"color":"brown","name":"Barbra Valenzuela","gender":"female","email":"barbravalenzuela@chorizon.com","phone":"+1 (992) 512-2649","address":"617 Schenck Court, Reinerton, Michigan, 2908","about":"Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n","registered":"2019-03-29T01:59:31 -01:00","latitude":45.193723,"longitude":-12.486778,"tags":["new issue","new issue","wontfix","wontfix"]}
+{"id":37,"isActive":false,"balance":"$3,469.82","picture":"http://placehold.it/32x32","age":39,"color":"brown","name":"Opal Weiss","gender":"female","email":"opalweiss@chorizon.com","phone":"+1 (809) 400-3079","address":"535 Bogart Street, Frizzleburg, Arizona, 5222","about":"Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n","registered":"2019-09-04T07:22:28 -02:00","latitude":72.50376,"longitude":61.656435,"tags":["bug","bug","good first issue","good first issue"]}
+{"id":38,"isActive":true,"balance":"$1,992.38","picture":"http://placehold.it/32x32","age":40,"color":"Green","name":"Christina Short","gender":"female","email":"christinashort@chorizon.com","phone":"+1 (884) 589-2705","address":"594 Willmohr Street, Dexter, Montana, 660","about":"Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n","registered":"2014-01-21T09:31:56 -01:00","latitude":-42.762739,"longitude":77.052349,"tags":["bug","new issue"]}
+{"id":39,"isActive":false,"balance":"$1,722.85","picture":"http://placehold.it/32x32","age":29,"color":"brown","name":"Golden Horton","gender":"male","email":"goldenhorton@chorizon.com","phone":"+1 (903) 426-2489","address":"191 Schenck Avenue, Mayfair, North Dakota, 5000","about":"Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n","registered":"2015-08-19T02:56:41 -02:00","latitude":69.922534,"longitude":9.881433,"tags":["bug"]}
+{"id":40,"isActive":false,"balance":"$1,656.54","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Stafford Emerson","gender":"male","email":"staffordemerson@chorizon.com","phone":"+1 (992) 455-2573","address":"523 Thornton Street, Conway, Vermont, 6331","about":"Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n","registered":"2019-02-16T04:07:08 -01:00","latitude":-29.143111,"longitude":-57.207703,"tags":["wontfix","good first issue","good first issue"]}
+{"id":41,"isActive":false,"balance":"$1,861.56","picture":"http://placehold.it/32x32","age":21,"color":"brown","name":"Salinas Gamble","gender":"male","email":"salinasgamble@chorizon.com","phone":"+1 (901) 525-2373","address":"991 Nostrand Avenue, Kansas, Mississippi, 6756","about":"Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n","registered":"2017-08-21T05:47:53 -02:00","latitude":-22.593819,"longitude":-63.613004,"tags":["good first issue","bug","bug","wontfix"]}
+{"id":42,"isActive":true,"balance":"$3,179.74","picture":"http://placehold.it/32x32","age":34,"color":"brown","name":"Graciela Russell","gender":"female","email":"gracielarussell@chorizon.com","phone":"+1 (893) 464-3951","address":"361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713","about":"Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n","registered":"2015-05-18T09:52:56 -02:00","latitude":-14.634444,"longitude":12.931783,"tags":["wontfix","bug","wontfix"]}
+{"id":43,"isActive":true,"balance":"$1,777.38","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Arnold Bender","gender":"male","email":"arnoldbender@chorizon.com","phone":"+1 (945) 581-3808","address":"781 Lorraine Street, Gallina, American Samoa, 1832","about":"Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n","registered":"2018-12-23T02:26:30 -01:00","latitude":41.208579,"longitude":51.948925,"tags":["bug","good first issue","good first issue","wontfix"]}
+{"id":44,"isActive":true,"balance":"$2,893.45","picture":"http://placehold.it/32x32","age":22,"color":"Green","name":"Joni Spears","gender":"female","email":"jonispears@chorizon.com","phone":"+1 (916) 565-2124","address":"307 Harwood Place, Canterwood, Maryland, 2047","about":"Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n","registered":"2015-03-01T12:38:28 -01:00","latitude":8.19071,"longitude":146.323808,"tags":["wontfix","new issue","good first issue","good first issue"]}
+{"id":45,"isActive":true,"balance":"$2,830.36","picture":"http://placehold.it/32x32","age":20,"color":"brown","name":"Irene Bennett","gender":"female","email":"irenebennett@chorizon.com","phone":"+1 (904) 431-2211","address":"353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686","about":"Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n","registered":"2018-04-17T05:18:51 -02:00","latitude":-36.435177,"longitude":-127.552573,"tags":["bug","wontfix"]}
+{"id":46,"isActive":true,"balance":"$1,348.04","picture":"http://placehold.it/32x32","age":34,"color":"Green","name":"Lawson Curtis","gender":"male","email":"lawsoncurtis@chorizon.com","phone":"+1 (896) 532-2172","address":"942 Gerritsen Avenue, Southmont, Kansas, 8915","about":"Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n","registered":"2016-08-23T01:41:09 -02:00","latitude":-48.783539,"longitude":20.492944,"tags":[]}
+{"id":47,"isActive":true,"balance":"$1,132.41","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goff May","gender":"male","email":"goffmay@chorizon.com","phone":"+1 (859) 453-3415","address":"225 Rutledge Street, Boonville, Massachusetts, 4081","about":"Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n","registered":"2014-10-25T07:32:30 -02:00","latitude":13.079225,"longitude":76.215086,"tags":["bug"]}
+{"id":48,"isActive":true,"balance":"$1,201.87","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goodman Becker","gender":"male","email":"goodmanbecker@chorizon.com","phone":"+1 (825) 470-3437","address":"388 Seigel Street, Sisquoc, Kentucky, 8231","about":"Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n","registered":"2019-09-05T04:49:03 -02:00","latitude":-23.792094,"longitude":-13.621221,"tags":["bug","bug","wontfix","new issue"]}
+{"id":49,"isActive":true,"balance":"$1,476.39","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Maureen Dale","gender":"female","email":"maureendale@chorizon.com","phone":"+1 (984) 538-3684","address":"817 Newton Street, Bannock, Wyoming, 1468","about":"Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n","registered":"2018-04-26T06:04:40 -02:00","latitude":-64.196802,"longitude":-117.396238,"tags":["wontfix"]}
+{"id":50,"isActive":true,"balance":"$1,947.08","picture":"http://placehold.it/32x32","age":21,"color":"Green","name":"Guerra Mcintyre","gender":"male","email":"guerramcintyre@chorizon.com","phone":"+1 (951) 536-2043","address":"423 Lombardy Street, Stewart, West Virginia, 908","about":"Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n","registered":"2015-07-16T05:11:42 -02:00","latitude":79.733743,"longitude":-20.602356,"tags":["bug","good first issue","good first issue"]}
+{"id":51,"isActive":true,"balance":"$2,960.90","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Key Cervantes","gender":"male","email":"keycervantes@chorizon.com","phone":"+1 (931) 474-3865","address":"410 Barbey Street, Vernon, Oregon, 2328","about":"Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n","registered":"2019-12-15T12:13:35 -01:00","latitude":47.627647,"longitude":117.049918,"tags":["new issue"]}
+{"id":52,"isActive":false,"balance":"$1,884.02","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Karen Nelson","gender":"female","email":"karennelson@chorizon.com","phone":"+1 (993) 528-3607","address":"930 Frank Court, Dunbar, New York, 8810","about":"Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n","registered":"2014-06-23T09:21:44 -02:00","latitude":-59.059033,"longitude":76.565373,"tags":["new issue","bug"]}
+{"id":53,"isActive":true,"balance":"$3,559.55","picture":"http://placehold.it/32x32","age":32,"color":"brown","name":"Caitlin Burnett","gender":"female","email":"caitlinburnett@chorizon.com","phone":"+1 (945) 480-2796","address":"516 Senator Street, Emory, Iowa, 4145","about":"In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n","registered":"2019-01-09T02:26:31 -01:00","latitude":-82.774237,"longitude":42.316194,"tags":["bug","good first issue"]}
+{"id":54,"isActive":true,"balance":"$2,113.29","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Richards Walls","gender":"male","email":"richardswalls@chorizon.com","phone":"+1 (865) 517-2982","address":"959 Brightwater Avenue, Stevens, Nevada, 2968","about":"Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n","registered":"2014-09-25T06:51:22 -02:00","latitude":80.09202,"longitude":87.49759,"tags":["wontfix","wontfix","bug"]}
+{"id":55,"isActive":true,"balance":"$1,977.66","picture":"http://placehold.it/32x32","age":36,"color":"brown","name":"Combs Stanley","gender":"male","email":"combsstanley@chorizon.com","phone":"+1 (827) 419-2053","address":"153 Beverley Road, Siglerville, South Carolina, 3666","about":"Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n","registered":"2019-08-22T07:53:15 -02:00","latitude":78.386181,"longitude":143.661058,"tags":[]}
+{"id":56,"isActive":false,"balance":"$3,886.12","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Tucker Barry","gender":"male","email":"tuckerbarry@chorizon.com","phone":"+1 (808) 544-3433","address":"805 Jamaica Avenue, Cornfields, Minnesota, 3689","about":"Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n","registered":"2016-08-29T07:28:00 -02:00","latitude":71.701551,"longitude":9.903068,"tags":[]}
+{"id":57,"isActive":false,"balance":"$1,844.56","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Kaitlin Conner","gender":"female","email":"kaitlinconner@chorizon.com","phone":"+1 (862) 467-2666","address":"501 Knight Court, Joppa, Rhode Island, 274","about":"Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n","registered":"2019-05-30T06:38:24 -02:00","latitude":15.613464,"longitude":171.965629,"tags":[]}
+{"id":58,"isActive":true,"balance":"$2,876.10","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Mamie Fischer","gender":"female","email":"mamiefischer@chorizon.com","phone":"+1 (948) 545-3901","address":"599 Hunterfly Place, Haena, Georgia, 6005","about":"Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n","registered":"2019-05-27T05:07:10 -02:00","latitude":70.915079,"longitude":-48.813584,"tags":["bug","wontfix","wontfix","good first issue"]}
+{"id":59,"isActive":true,"balance":"$1,921.58","picture":"http://placehold.it/32x32","age":31,"color":"Green","name":"Harper Carson","gender":"male","email":"harpercarson@chorizon.com","phone":"+1 (912) 430-3243","address":"883 Dennett Place, Knowlton, New Mexico, 9219","about":"Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n","registered":"2019-12-07T07:33:15 -01:00","latitude":-60.812605,"longitude":-27.129016,"tags":["bug","new issue"]}
+{"id":60,"isActive":true,"balance":"$1,770.93","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Jody Herrera","gender":"female","email":"jodyherrera@chorizon.com","phone":"+1 (890) 583-3222","address":"261 Jay Street, Strykersville, Ohio, 9248","about":"Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n","registered":"2016-05-21T01:00:02 -02:00","latitude":-36.846586,"longitude":131.156223,"tags":[]}
+{"id":61,"isActive":false,"balance":"$2,813.41","picture":"http://placehold.it/32x32","age":37,"color":"Green","name":"Charles Castillo","gender":"male","email":"charlescastillo@chorizon.com","phone":"+1 (934) 467-2108","address":"675 Morton Street, Rew, Pennsylvania, 137","about":"Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n","registered":"2019-06-10T02:54:22 -02:00","latitude":-16.423202,"longitude":-146.293752,"tags":["new issue","new issue"]}
+{"id":62,"isActive":true,"balance":"$3,341.35","picture":"http://placehold.it/32x32","age":33,"color":"blue","name":"Estelle Ramirez","gender":"female","email":"estelleramirez@chorizon.com","phone":"+1 (816) 459-2073","address":"636 Nolans Lane, Camptown, California, 7794","about":"Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n","registered":"2015-02-14T01:05:50 -01:00","latitude":-46.591249,"longitude":-83.385587,"tags":["good first issue","bug"]}
+{"id":63,"isActive":true,"balance":"$2,478.30","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Knowles Hebert","gender":"male","email":"knowleshebert@chorizon.com","phone":"+1 (819) 409-2308","address":"361 Kathleen Court, Gratton, Connecticut, 7254","about":"Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n","registered":"2016-03-08T08:34:52 -01:00","latitude":71.042482,"longitude":152.460406,"tags":["good first issue","wontfix"]}
+{"id":64,"isActive":false,"balance":"$2,559.09","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Thelma Mckenzie","gender":"female","email":"thelmamckenzie@chorizon.com","phone":"+1 (941) 596-2777","address":"202 Leonard Street, Riverton, Illinois, 8577","about":"Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n","registered":"2020-04-14T12:43:06 -02:00","latitude":16.026129,"longitude":105.464476,"tags":[]}
+{"id":65,"isActive":true,"balance":"$1,025.08","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Carole Rowland","gender":"female","email":"carolerowland@chorizon.com","phone":"+1 (862) 558-3448","address":"941 Melba Court, Bluetown, Florida, 9555","about":"Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n","registered":"2014-12-01T05:55:35 -01:00","latitude":-0.191998,"longitude":43.389652,"tags":["wontfix"]}
+{"id":66,"isActive":true,"balance":"$1,061.49","picture":"http://placehold.it/32x32","age":35,"color":"brown","name":"Higgins Aguilar","gender":"male","email":"higginsaguilar@chorizon.com","phone":"+1 (911) 540-3791","address":"132 Sackman Street, Layhill, Guam, 8729","about":"Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n","registered":"2015-04-05T02:10:07 -02:00","latitude":74.702813,"longitude":151.314972,"tags":["bug"]}
+{"id":67,"isActive":true,"balance":"$3,510.14","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Ilene Gillespie","gender":"female","email":"ilenegillespie@chorizon.com","phone":"+1 (937) 575-2676","address":"835 Lake Street, Naomi, Alabama, 4131","about":"Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n","registered":"2015-06-28T09:41:45 -02:00","latitude":71.573342,"longitude":-95.295989,"tags":["wontfix","wontfix"]}
+{"id":68,"isActive":false,"balance":"$1,539.98","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Angelina Dyer","gender":"female","email":"angelinadyer@chorizon.com","phone":"+1 (948) 574-3949","address":"575 Division Place, Gorham, Louisiana, 3458","about":"Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n","registered":"2014-07-08T06:34:36 -02:00","latitude":-85.649593,"longitude":66.126018,"tags":["good first issue"]}
+{"id":69,"isActive":true,"balance":"$3,367.69","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Marks Burt","gender":"male","email":"marksburt@chorizon.com","phone":"+1 (895) 497-3138","address":"819 Village Road, Wadsworth, Delaware, 6099","about":"Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n","registered":"2014-08-31T06:12:18 -02:00","latitude":26.854112,"longitude":-143.313948,"tags":["good first issue"]}
+{"id":70,"isActive":false,"balance":"$3,755.72","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Glass Perkins","gender":"male","email":"glassperkins@chorizon.com","phone":"+1 (923) 486-3725","address":"899 Roosevelt Court, Belleview, Idaho, 1737","about":"Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n","registered":"2015-05-22T05:44:33 -02:00","latitude":54.27147,"longitude":-65.065604,"tags":["wontfix"]}
+{"id":71,"isActive":true,"balance":"$3,381.63","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Candace Sawyer","gender":"female","email":"candacesawyer@chorizon.com","phone":"+1 (830) 404-2636","address":"334 Arkansas Drive, Bordelonville, Tennessee, 8449","about":"Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n","registered":"2014-04-04T08:45:00 -02:00","latitude":6.484262,"longitude":-37.054928,"tags":["new issue","new issue"]}
+{"id":72,"isActive":true,"balance":"$1,640.98","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Hendricks Martinez","gender":"male","email":"hendricksmartinez@chorizon.com","phone":"+1 (857) 566-3245","address":"636 Agate Court, Newry, Utah, 3304","about":"Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n","registered":"2018-06-15T10:36:11 -02:00","latitude":86.746034,"longitude":10.347893,"tags":["new issue"]}
+{"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]}
+{"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]}
+{"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]}
+{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]}

meilisearch-http/tests/assets/dumps/v1/test/settings.json

@@ -0,0 +1,59 @@
+{
+    "rankingRules": [
+        "typo",
+        "words",
+        "proximity",
+        "attribute",
+        "wordsPosition",
+        "exactness"
+    ],
+    "distinctAttribute": "email",
+    "searchableAttributes": [
+        "balance",
+        "picture",
+        "age",
+        "color",
+        "name",
+        "gender",
+        "email",
+        "phone",
+        "address",
+        "about",
+        "registered",
+        "latitude",
+        "longitude",
+        "tags"
+    ],
+    "displayedAttributes": [
+        "id",
+        "isActive",
+        "balance",
+        "picture",
+        "age",
+        "color",
+        "name",
+        "gender",
+        "email",
+        "phone",
+        "address",
+        "about",
+        "registered",
+        "latitude",
+        "longitude",
+        "tags"
+    ],
+    "stopWords": [
+        "in",
+        "ad"
+    ],
+    "synonyms": {
+        "wolverine": ["xmen", "logan"],
+        "logan": ["wolverine", "xmen"]
+    },
+    "filterableAttributes": [
+        "gender",
+        "color",
+        "tags",
+        "isActive"
+    ]
+}

meilisearch-http/tests/assets/dumps/v1/test/updates.jsonl

@@ -0,0 +1,2 @@
+{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","filterable_attributes":"Nothing"}}}
+{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}}

meilisearch-http/tests/common/index.rs

@@ -0,0 +1,198 @@
+use std::time::Duration;
+
+use actix_web::http::StatusCode;
+use paste::paste;
+use serde_json::{json, Value};
+use tokio::time::sleep;
+
+use super::service::Service;
+
+macro_rules! make_settings_test_routes {
+    ($($name:ident),+) => {
+        $(paste! {
+            pub async fn [<update_$name>](&self, value: Value) -> (Value, StatusCode) {
+                let url = format!("/indexes/{}/settings/{}", self.uid, stringify!($name).replace("_", "-"));
+                self.service.post(url, value).await
+            }
+
+            pub async fn [<get_$name>](&self) -> (Value, StatusCode) {
+                let url = format!("/indexes/{}/settings/{}", self.uid, stringify!($name).replace("_", "-"));
+                self.service.get(url).await
+            }
+        })*
+    };
+}
+
+pub struct Index<'a> {
+    pub uid: String,
+    pub service: &'a Service,
+}
+
+#[allow(dead_code)]
+impl Index<'_> {
+    pub async fn get(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", self.uid);
+        self.service.get(url).await
+    }
+
+    pub async fn load_test_set(&self) -> u64 {
+        let url = format!("/indexes/{}/documents", self.uid);
+        let (response, code) = self
+            .service
+            .post_str(url, include_str!("../assets/test_set.json"))
+            .await;
+        assert_eq!(code, 202);
+        let update_id = response["updateId"].as_i64().unwrap();
+        self.wait_update_id(update_id as u64).await;
+        update_id as u64
+    }
+
+    pub async fn create(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
+        let body = json!({
+            "uid": self.uid,
+            "primaryKey": primary_key,
+        });
+        self.service.post("/indexes", body).await
+    }
+
+    pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
+        let body = json!({
+            "primaryKey": primary_key,
+        });
+        let url = format!("/indexes/{}", self.uid);
+
+        self.service.put(url, body).await
+    }
+
+    pub async fn delete(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", self.uid);
+        self.service.delete(url).await
+    }
+
+    pub async fn add_documents(
+        &self,
+        documents: Value,
+        primary_key: Option<&str>,
+    ) -> (Value, StatusCode) {
+        let url = match primary_key {
+            Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key),
+            None => format!("/indexes/{}/documents", self.uid),
+        };
+        self.service.post(url, documents).await
+    }
+
+    pub async fn update_documents(
+        &self,
+        documents: Value,
+        primary_key: Option<&str>,
+    ) -> (Value, StatusCode) {
+        let url = match primary_key {
+            Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key),
+            None => format!("/indexes/{}/documents", self.uid),
+        };
+        self.service.put(url, documents).await
+    }
+
+    pub async fn wait_update_id(&self, update_id: u64) -> Value {
+        // try 10 times to get status, or panic to not wait forever
+        let url = format!("/indexes/{}/updates/{}", self.uid, update_id);
+        for _ in 0..10 {
+            let (response, status_code) = self.service.get(&url).await;
+            assert_eq!(status_code, 200, "response: {}", response);
+
+            if response["status"] == "processed" || response["status"] == "failed" {
+                return response;
+            }
+
+            sleep(Duration::from_secs(1)).await;
+        }
+        panic!("Timeout waiting for update id");
+    }
+
+    pub async fn get_update(&self, update_id: u64) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/updates/{}", self.uid, update_id);
+        self.service.get(url).await
+    }
+
+    pub async fn list_updates(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/updates", self.uid);
+        self.service.get(url).await
+    }
+
+    pub async fn get_document(
+        &self,
+        id: u64,
+        _options: Option<GetDocumentOptions>,
+    ) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents/{}", self.uid, id);
+        self.service.get(url).await
+    }
+
+    pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
+        let mut url = format!("/indexes/{}/documents?", self.uid);
+        if let Some(limit) = options.limit {
+            url.push_str(&format!("limit={}&", limit));
+        }
+
+        if let Some(offset) = options.offset {
+            url.push_str(&format!("offset={}&", offset));
+        }
+
+        if let Some(attributes_to_retrieve) = options.attributes_to_retrieve {
+            url.push_str(&format!(
+                "attributesToRetrieve={}&",
+                attributes_to_retrieve.join(",")
+            ));
+        }
+
+        self.service.get(url).await
+    }
+
+    pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents/{}", self.uid, id);
+        self.service.delete(url).await
+    }
+
+    pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents", self.uid);
+        self.service.delete(url).await
+    }
+
+    pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents/delete-batch", self.uid);
+        self.service
+            .post(url, serde_json::to_value(&ids).unwrap())
+            .await
+    }
+
+    pub async fn settings(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", self.uid);
+        self.service.get(url).await
+    }
+
+    pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", self.uid);
+        self.service.post(url, settings).await
+    }
+
+    pub async fn delete_settings(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", self.uid);
+        self.service.delete(url).await
+    }
+
+    pub async fn stats(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/stats", self.uid);
+        self.service.get(url).await
+    }
+
+    make_settings_test_routes!(distinct_attribute);
+}
+
+pub struct GetDocumentOptions;
+
+#[derive(Debug, Default)]
+pub struct GetAllDocumentsOptions {
+    pub limit: Option<usize>,
+    pub offset: Option<usize>,
+    pub attributes_to_retrieve: Option<Vec<&'static str>>,
+}

meilisearch-http/tests/common/mod.rs

@@ -0,0 +1,31 @@
+pub mod index;
+pub mod server;
+pub mod service;
+
+pub use index::{GetAllDocumentsOptions, GetDocumentOptions};
+pub use server::Server;
+
+/// Performs a search test on both post and get routes
+#[macro_export]
+macro_rules! test_post_get_search {
+    ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => {
+        let post_query: meilisearch_http::routes::search::SearchQueryPost =
+            serde_json::from_str(&$query.clone().to_string()).unwrap();
+        let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into();
+        let get_query = ::serde_url_params::to_string(&get_query).unwrap();
+        let ($response, $status_code) = $server.search_get(&get_query).await;
+        let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| {
+            panic!(
+                "panic in get route: {:?}",
+                e.downcast_ref::<&str>().unwrap()
+            )
+        });
+        let ($response, $status_code) = $server.search_post($query).await;
+        let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| {
+            panic!(
+                "panic in post route: {:?}",
+                e.downcast_ref::<&str>().unwrap()
+            )
+        });
+    };
+}

meilisearch-http/tests/common/server.rs

@@ -0,0 +1,96 @@
+use std::path::Path;
+
+use actix_web::http::StatusCode;
+use byte_unit::{Byte, ByteUnit};
+use serde_json::Value;
+use tempdir::TempDir;
+use urlencoding::encode;
+
+use meilisearch_http::data::Data;
+use meilisearch_http::option::{IndexerOpts, Opt};
+
+use super::index::Index;
+use super::service::Service;
+
+pub struct Server {
+    pub service: Service,
+    // hold ownership to the tempdir while we use the server instance.
+    _dir: Option<tempdir::TempDir>,
+}
+
+impl Server {
+    pub async fn new() -> Self {
+        let dir = TempDir::new("meilisearch").unwrap();
+
+        let opt = default_settings(dir.path());
+
+        let data = Data::new(opt).unwrap();
+        let service = Service(data);
+
+        Server {
+            service,
+            _dir: Some(dir),
+        }
+    }
+
+    pub async fn new_with_options(opt: Opt) -> Self {
+        let data = Data::new(opt).unwrap();
+        let service = Service(data);
+
+        Server {
+            service,
+            _dir: None,
+        }
+    }
+
+    /// Returns a view to an index. There is no guarantee that the index exists.
+    pub fn index(&self, uid: impl AsRef<str>) -> Index<'_> {
+        Index {
+            uid: encode(uid.as_ref()),
+            service: &self.service,
+        }
+    }
+
+    pub async fn list_indexes(&self) -> (Value, StatusCode) {
+        self.service.get("/indexes").await
+    }
+
+    pub async fn version(&self) -> (Value, StatusCode) {
+        self.service.get("/version").await
+    }
+
+    pub async fn stats(&self) -> (Value, StatusCode) {
+        self.service.get("/stats").await
+    }
+}
+
+pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
+    Opt {
+        db_path: dir.as_ref().join("db"),
+        dumps_dir: dir.as_ref().join("dump"),
+        http_addr: "127.0.0.1:7700".to_owned(),
+        master_key: None,
+        env: "development".to_owned(),
+        #[cfg(all(not(debug_assertions), feature = "analytics"))]
+        no_analytics: true,
+        max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
+        max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
+        http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(),
+        ssl_cert_path: None,
+        ssl_key_path: None,
+        ssl_auth_path: None,
+        ssl_ocsp_path: None,
+        ssl_require_auth: false,
+        ssl_resumption: false,
+        ssl_tickets: false,
+        import_snapshot: None,
+        ignore_missing_snapshot: false,
+        ignore_snapshot_if_db_exists: false,
+        snapshot_dir: ".".into(),
+        schedule_snapshot: false,
+        snapshot_interval_sec: 0,
+        import_dump: None,
+        indexer_options: IndexerOpts::default(),
+        log_level: "off".into(),
+    }
+}

meilisearch-http/tests/common/service.rs

@@ -0,0 +1,84 @@
+use actix_web::{http::StatusCode, test};
+use serde_json::Value;
+
+use meilisearch_http::create_app;
+use meilisearch_http::data::Data;
+
+pub struct Service(pub Data);
+
+impl Service {
+    pub async fn post(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
+        let app = test::init_service(create_app!(&self.0, true)).await;
+
+        let req = test::TestRequest::post()
+            .uri(url.as_ref())
+            .set_json(&body)
+            .to_request();
+        let res = test::call_service(&app, req).await;
+        let status_code = res.status();
+
+        let body = test::read_body(res).await;
+        let response = serde_json::from_slice(&body).unwrap_or_default();
+        (response, status_code)
+    }
+
+    /// Send a test post request from a text body, with a `content-type:application/json` header.
+    pub async fn post_str(
+        &self,
+        url: impl AsRef<str>,
+        body: impl AsRef<str>,
+    ) -> (Value, StatusCode) {
+        let app = test::init_service(create_app!(&self.0, true)).await;
+
+        let req = test::TestRequest::post()
+            .uri(url.as_ref())
+            .set_payload(body.as_ref().to_string())
+            .insert_header(("content-type", "application/json"))
+            .to_request();
+        let res = test::call_service(&app, req).await;
+        let status_code = res.status();
+
+        let body = test::read_body(res).await;
+        let response = serde_json::from_slice(&body).unwrap_or_default();
+        (response, status_code)
+    }
+
+    pub async fn get(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
+        let app = test::init_service(create_app!(&self.0, true)).await;
+
+        let req = test::TestRequest::get().uri(url.as_ref()).to_request();
+        let res = test::call_service(&app, req).await;
+        let status_code = res.status();
+
+        let body = test::read_body(res).await;
+        let response = serde_json::from_slice(&body).unwrap_or_default();
+        (response, status_code)
+    }
+
+    pub async fn put(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
+        let app = test::init_service(create_app!(&self.0, true)).await;
+
+        let req = test::TestRequest::put()
+            .uri(url.as_ref())
+            .set_json(&body)
+            .to_request();
+        let res = test::call_service(&app, req).await;
+        let status_code = res.status();
+
+        let body = test::read_body(res).await;
+        let response = serde_json::from_slice(&body).unwrap_or_default();
+        (response, status_code)
+    }
+
+    pub async fn delete(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
+        let app = test::init_service(create_app!(&self.0, true)).await;
+
+        let req = test::TestRequest::delete().uri(url.as_ref()).to_request();
+        let res = test::call_service(&app, req).await;
+        let status_code = res.status();
+
+        let body = test::read_body(res).await;
+        let response = serde_json::from_slice(&body).unwrap_or_default();
+        (response, status_code)
+    }
+}

meilisearch-http/tests/documents/add_documents.rs

@@ -0,0 +1,428 @@
+use crate::common::{GetAllDocumentsOptions, Server};
+use actix_web::test;
+use chrono::DateTime;
+use meilisearch_http::create_app;
+use serde_json::{json, Value};
+
+/// This is the basic usage of our API and every other tests uses the content-type application/json
+#[actix_rt::test]
+async fn add_documents_test_json_content_types() {
+    let document = json!([
+        {
+            "id": 1,
+            "content": "Bouvier Bernois",
+        }
+    ]);
+
+    // this is a what is expected and should work
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(&server.service.0, true)).await;
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/json"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 202);
+    assert_eq!(response, json!({ "updateId": 0 }));
+}
+
+/// no content type is still supposed to be accepted as json
+#[actix_rt::test]
+async fn add_documents_test_no_content_types() {
+    let document = json!([
+        {
+            "id": 1,
+            "content": "Montagne des Pyrénées",
+        }
+    ]);
+
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(&server.service.0, true)).await;
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/json"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 202);
+    assert_eq!(response, json!({ "updateId": 0 }));
+}
+
+/// any other content-type is must be refused
+#[actix_rt::test]
+async fn add_documents_test_bad_content_types() {
+    let document = json!([
+        {
+            "id": 1,
+            "content": "Leonberg",
+        }
+    ]);
+
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(&server.service.0, true)).await;
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "text/plain"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    assert_eq!(status_code, 405);
+    assert!(body.is_empty());
+}
+
+#[actix_rt::test]
+async fn add_documents_no_index_creation() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        {
+            "id": 1,
+            "content": "foo",
+        }
+    ]);
+
+    let (response, code) = index.add_documents(documents, None).await;
+    assert_eq!(code, 202);
+    assert_eq!(response["updateId"], 0);
+    /*
+     * currently we don’t check these field to stay ISO with meilisearch
+     * assert_eq!(response["status"], "pending");
+     * assert_eq!(response["meta"]["type"], "DocumentsAddition");
+     * assert_eq!(response["meta"]["format"], "Json");
+     * assert_eq!(response["meta"]["primaryKey"], Value::Null);
+     * assert!(response.get("enqueuedAt").is_some());
+     */
+
+    index.wait_update_id(0).await;
+
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "processed");
+    assert_eq!(response["updateId"], 0);
+    assert_eq!(response["type"]["name"], "DocumentsAddition");
+    assert_eq!(response["type"]["number"], 1);
+
+    let processed_at =
+        DateTime::parse_from_rfc3339(response["processedAt"].as_str().unwrap()).unwrap();
+    let enqueued_at =
+        DateTime::parse_from_rfc3339(response["enqueuedAt"].as_str().unwrap()).unwrap();
+    assert!(processed_at > enqueued_at);
+
+    // index was created, and primary key was infered.
+    let (response, code) = index.get().await;
+    assert_eq!(code, 200);
+    assert_eq!(response["primaryKey"], "id");
+}
+
+#[actix_rt::test]
+async fn document_add_create_index_bad_uid() {
+    let server = Server::new().await;
+    let index = server.index("883  fj!");
+    let (_response, code) = index.add_documents(json!([]), None).await;
+    assert_eq!(code, 400);
+}
+
+#[actix_rt::test]
+async fn document_update_create_index_bad_uid() {
+    let server = Server::new().await;
+    let index = server.index("883  fj!");
+    let (_response, code) = index.update_documents(json!([]), None).await;
+    assert_eq!(code, 400);
+}
+
+#[actix_rt::test]
+async fn document_addition_with_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        {
+            "primary": 1,
+            "content": "foo",
+        }
+    ]);
+    let (response, code) = index.add_documents(documents, Some("primary")).await;
+    assert_eq!(code, 202, "response: {}", response);
+
+    index.wait_update_id(0).await;
+
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "processed");
+    assert_eq!(response["updateId"], 0);
+    assert_eq!(response["type"]["name"], "DocumentsAddition");
+    assert_eq!(response["type"]["number"], 1);
+
+    let (response, code) = index.get().await;
+    assert_eq!(code, 200);
+    assert_eq!(response["primaryKey"], "primary");
+}
+
+#[actix_rt::test]
+async fn document_update_with_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        {
+            "primary": 1,
+            "content": "foo",
+        }
+    ]);
+    let (_response, code) = index.update_documents(documents, Some("primary")).await;
+    assert_eq!(code, 202);
+
+    index.wait_update_id(0).await;
+
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "processed");
+    assert_eq!(response["updateId"], 0);
+    assert_eq!(response["type"]["name"], "DocumentsPartial");
+    assert_eq!(response["type"]["number"], 1);
+
+    let (response, code) = index.get().await;
+    assert_eq!(code, 200);
+    assert_eq!(response["primaryKey"], "primary");
+}
+
+#[actix_rt::test]
+async fn add_documents_with_primary_key_and_primary_key_already_exists() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index.create(Some("primary")).await;
+    let documents = json!([
+        {
+            "id": 1,
+            "content": "foo",
+        }
+    ]);
+
+    let (_response, code) = index.add_documents(documents, Some("id")).await;
+    assert_eq!(code, 202);
+
+    index.wait_update_id(0).await;
+
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "failed");
+
+    let (response, code) = index.get().await;
+    assert_eq!(code, 200);
+    assert_eq!(response["primaryKey"], "primary");
+}
+
+#[actix_rt::test]
+async fn update_documents_with_primary_key_and_primary_key_already_exists() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index.create(Some("primary")).await;
+    let documents = json!([
+        {
+            "id": 1,
+            "content": "foo",
+        }
+    ]);
+
+    let (_response, code) = index.update_documents(documents, Some("id")).await;
+    assert_eq!(code, 202);
+
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    // Documents without a primary key are not accepted.
+    assert_eq!(response["status"], "failed");
+
+    let (response, code) = index.get().await;
+    assert_eq!(code, 200);
+    assert_eq!(response["primaryKey"], "primary");
+}
+
+#[actix_rt::test]
+async fn replace_document() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        {
+            "doc_id": 1,
+            "content": "foo",
+        }
+    ]);
+
+    let (response, code) = index.add_documents(documents, None).await;
+    assert_eq!(code, 202, "response: {}", response);
+
+    index.wait_update_id(0).await;
+
+    let documents = json!([
+        {
+            "doc_id": 1,
+            "other": "bar",
+        }
+    ]);
+
+    let (_response, code) = index.add_documents(documents, None).await;
+    assert_eq!(code, 202);
+
+    index.wait_update_id(1).await;
+
+    let (response, code) = index.get_update(1).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "processed");
+
+    let (response, code) = index.get_document(1, None).await;
+    assert_eq!(code, 200);
+    assert_eq!(response.to_string(), r##"{"doc_id":1,"other":"bar"}"##);
+}
+
+// test broken, see issue milli#92
+#[actix_rt::test]
+#[ignore]
+async fn add_no_documents() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_response, code) = index.add_documents(json!([]), None).await;
+    assert_eq!(code, 200);
+
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "processed");
+    assert_eq!(response["updateId"], 0);
+    assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 0);
+
+    let (response, code) = index.get().await;
+    assert_eq!(code, 200);
+    assert_eq!(response["primaryKey"], Value::Null);
+}
+
+#[actix_rt::test]
+async fn update_document() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        {
+            "doc_id": 1,
+            "content": "foo",
+        }
+    ]);
+
+    let (_response, code) = index.add_documents(documents, None).await;
+    assert_eq!(code, 202);
+
+    index.wait_update_id(0).await;
+
+    let documents = json!([
+        {
+            "doc_id": 1,
+            "other": "bar",
+        }
+    ]);
+
+    let (response, code) = index.update_documents(documents, None).await;
+    assert_eq!(code, 202, "response: {}", response);
+
+    index.wait_update_id(1).await;
+
+    let (response, code) = index.get_update(1).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "processed");
+
+    let (response, code) = index.get_document(1, None).await;
+    assert_eq!(code, 200);
+    assert_eq!(
+        response.to_string(),
+        r##"{"doc_id":1,"content":"foo","other":"bar"}"##
+    );
+}
+
+#[actix_rt::test]
+async fn add_larger_dataset() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let update_id = index.load_test_set().await;
+    let (response, code) = index.get_update(update_id).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "processed");
+    assert_eq!(response["type"]["name"], "DocumentsAddition");
+    assert_eq!(response["type"]["number"], 77);
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            limit: Some(1000),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 77);
+}
+
+#[actix_rt::test]
+async fn update_larger_dataset() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let documents = serde_json::from_str(include_str!("../assets/test_set.json")).unwrap();
+    index.update_documents(documents, None).await;
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["type"]["name"], "DocumentsPartial");
+    assert_eq!(response["type"]["number"], 77);
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            limit: Some(1000),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 77);
+}
+
+#[actix_rt::test]
+async fn add_documents_bad_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(Some("docid")).await;
+    let documents = json!([
+        {
+            "docid": "foo & bar",
+            "content": "foobar"
+        }
+    ]);
+    index.add_documents(documents, None).await;
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "failed");
+}
+
+#[actix_rt::test]
+async fn update_documents_bad_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(Some("docid")).await;
+    let documents = json!([
+        {
+            "docid": "foo & bar",
+            "content": "foobar"
+        }
+    ]);
+    index.update_documents(documents, None).await;
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "failed");
+}

meilisearch-http/tests/documents/delete_documents.rs

@@ -0,0 +1,125 @@
+use serde_json::json;
+
+use crate::common::{GetAllDocumentsOptions, Server};
+
+#[actix_rt::test]
+async fn delete_one_document_unexisting_index() {
+    let server = Server::new().await;
+    let (_response, code) = server.index("test").delete_document(0).await;
+    assert_eq!(code, 404);
+}
+
+#[actix_rt::test]
+async fn delete_one_unexisting_document() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(None).await;
+    let (response, code) = index.delete_document(0).await;
+    assert_eq!(code, 202, "{}", response);
+    let update = index.wait_update_id(0).await;
+    assert_eq!(update["status"], "processed");
+}
+
+#[actix_rt::test]
+async fn delete_one_document() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index
+        .add_documents(json!([{ "id": 0, "content": "foobar" }]), None)
+        .await;
+    index.wait_update_id(0).await;
+    let (_response, code) = server.index("test").delete_document(0).await;
+    assert_eq!(code, 202);
+    index.wait_update_id(1).await;
+
+    let (_response, code) = index.get_document(0, None).await;
+    assert_eq!(code, 404);
+}
+
+#[actix_rt::test]
+async fn clear_all_documents_unexisting_index() {
+    let server = Server::new().await;
+    let (_response, code) = server.index("test").clear_all_documents().await;
+    assert_eq!(code, 404);
+}
+
+#[actix_rt::test]
+async fn clear_all_documents() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index
+        .add_documents(
+            json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }]),
+            None,
+        )
+        .await;
+    index.wait_update_id(0).await;
+    let (_response, code) = index.clear_all_documents().await;
+    assert_eq!(code, 202);
+
+    let _update = index.wait_update_id(1).await;
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions::default())
+        .await;
+    assert_eq!(code, 200);
+    assert!(response.as_array().unwrap().is_empty());
+}
+
+#[actix_rt::test]
+async fn clear_all_documents_empty_index() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(None).await;
+
+    let (_response, code) = index.clear_all_documents().await;
+    assert_eq!(code, 202);
+
+    let _update = index.wait_update_id(0).await;
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions::default())
+        .await;
+    assert_eq!(code, 200);
+    assert!(response.as_array().unwrap().is_empty());
+}
+
+#[actix_rt::test]
+async fn delete_batch_unexisting_index() {
+    let server = Server::new().await;
+    let (response, code) = server.index("test").delete_batch(vec![]).await;
+    assert_eq!(code, 404, "{}", response);
+}
+
+#[actix_rt::test]
+async fn delete_batch() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await;
+    index.wait_update_id(0).await;
+    let (_response, code) = index.delete_batch(vec![1, 0]).await;
+    assert_eq!(code, 202);
+
+    let _update = index.wait_update_id(1).await;
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions::default())
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 1);
+    assert_eq!(response.as_array().unwrap()[0]["id"], 3);
+}
+
+#[actix_rt::test]
+async fn delete_no_document_batch() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.add_documents(json!([{ "id": 1, "content": "foobar" }, { "id": 0, "content": "foobar" }, { "id": 3, "content": "foobar" }]), Some("id")).await;
+    index.wait_update_id(0).await;
+    let (_response, code) = index.delete_batch(vec![]).await;
+    assert_eq!(code, 202, "{}", _response);
+
+    let _update = index.wait_update_id(1).await;
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions::default())
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 3);
+}

meilisearch-http/tests/documents/get_documents.rs

@@ -0,0 +1,286 @@
+use crate::common::GetAllDocumentsOptions;
+use crate::common::Server;
+
+use serde_json::json;
+
+// TODO: partial test since we are testing error, amd error is not yet fully implemented in
+// transplant
+#[actix_rt::test]
+async fn get_unexisting_index_single_document() {
+    let server = Server::new().await;
+    let (_response, code) = server.index("test").get_document(1, None).await;
+    assert_eq!(code, 404);
+}
+
+#[actix_rt::test]
+async fn get_unexisting_document() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(None).await;
+    let (_response, code) = index.get_document(1, None).await;
+    assert_eq!(code, 404);
+}
+
+#[actix_rt::test]
+async fn get_document() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(None).await;
+    let documents = serde_json::json!([
+        {
+            "id": 0,
+            "content": "foobar",
+        }
+    ]);
+    let (_, code) = index.add_documents(documents, None).await;
+    assert_eq!(code, 202);
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_document(0, None).await;
+    assert_eq!(code, 200);
+    assert_eq!(
+        response,
+        serde_json::json!(        {
+            "id": 0,
+            "content": "foobar",
+        })
+    );
+}
+
+#[actix_rt::test]
+async fn get_unexisting_index_all_documents() {
+    let server = Server::new().await;
+    let (_response, code) = server
+        .index("test")
+        .get_all_documents(GetAllDocumentsOptions::default())
+        .await;
+    assert_eq!(code, 404);
+}
+
+#[actix_rt::test]
+async fn get_no_documents() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_, code) = index.create(None).await;
+    assert_eq!(code, 200);
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions::default())
+        .await;
+    assert_eq!(code, 200);
+    assert!(response.as_array().unwrap().is_empty());
+}
+
+#[actix_rt::test]
+async fn get_all_documents_no_options() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.load_test_set().await;
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions::default())
+        .await;
+    assert_eq!(code, 200);
+    let arr = response.as_array().unwrap();
+    assert_eq!(arr.len(), 20);
+    let first = serde_json::json!({
+        "id":0,
+        "isActive":false,
+        "balance":"$2,668.55",
+        "picture":"http://placehold.it/32x32",
+        "age":36,
+        "color":"Green",
+        "name":"Lucas Hess",
+        "gender":"male",
+        "email":"lucashess@chorizon.com",
+        "phone":"+1 (998) 478-2597",
+        "address":"412 Losee Terrace, Blairstown, Georgia, 2825",
+        "about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n",
+        "registered":"2016-06-21T09:30:25 -02:00",
+        "latitude":-44.174957,
+        "longitude":-145.725388,
+        "tags":["bug"
+            ,"bug"]});
+    assert_eq!(first, arr[0]);
+}
+
+#[actix_rt::test]
+async fn test_get_all_documents_limit() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.load_test_set().await;
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            limit: Some(5),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 5);
+    assert_eq!(response.as_array().unwrap()[0]["id"], 0);
+}
+
+#[actix_rt::test]
+async fn test_get_all_documents_offset() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.load_test_set().await;
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            offset: Some(5),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(response.as_array().unwrap()[0]["id"], 13);
+}
+
+#[actix_rt::test]
+async fn test_get_all_documents_attributes_to_retrieve() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.load_test_set().await;
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            attributes_to_retrieve: Some(vec!["name"]),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(
+        response.as_array().unwrap()[0]
+            .as_object()
+            .unwrap()
+            .keys()
+            .count(),
+        1
+    );
+    assert!(response.as_array().unwrap()[0]
+        .as_object()
+        .unwrap()
+        .get("name")
+        .is_some());
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            attributes_to_retrieve: Some(vec![]),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(
+        response.as_array().unwrap()[0]
+            .as_object()
+            .unwrap()
+            .keys()
+            .count(),
+        0
+    );
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            attributes_to_retrieve: Some(vec!["wrong"]),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(
+        response.as_array().unwrap()[0]
+            .as_object()
+            .unwrap()
+            .keys()
+            .count(),
+        0
+    );
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            attributes_to_retrieve: Some(vec!["name", "tags"]),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(
+        response.as_array().unwrap()[0]
+            .as_object()
+            .unwrap()
+            .keys()
+            .count(),
+        2
+    );
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            attributes_to_retrieve: Some(vec!["*"]),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(
+        response.as_array().unwrap()[0]
+            .as_object()
+            .unwrap()
+            .keys()
+            .count(),
+        16
+    );
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            attributes_to_retrieve: Some(vec!["*", "wrong"]),
+            ..Default::default()
+        })
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(
+        response.as_array().unwrap()[0]
+            .as_object()
+            .unwrap()
+            .keys()
+            .count(),
+        16
+    );
+}
+
+#[actix_rt::test]
+async fn get_documents_displayed_attributes() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index
+        .update_settings(json!({"displayedAttributes": ["gender"]}))
+        .await;
+    index.load_test_set().await;
+
+    let (response, code) = index
+        .get_all_documents(GetAllDocumentsOptions::default())
+        .await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_array().unwrap().len(), 20);
+    assert_eq!(
+        response.as_array().unwrap()[0]
+            .as_object()
+            .unwrap()
+            .keys()
+            .count(),
+        1
+    );
+    assert!(response.as_array().unwrap()[0]
+        .as_object()
+        .unwrap()
+        .get("gender")
+        .is_some());
+
+    let (response, code) = index.get_document(0, None).await;
+    assert_eq!(code, 200);
+    assert_eq!(response.as_object().unwrap().keys().count(), 1);
+    assert!(response.as_object().unwrap().get("gender").is_some());
+}

meilisearch-http/tests/documents/mod.rs

@@ -0,0 +1,3 @@
+mod add_documents;
+mod delete_documents;
+mod get_documents;

meilisearch-http/tests/index/create_index.rs

@@ -0,0 +1,74 @@
+use crate::common::Server;
+use serde_json::Value;
+
+#[actix_rt::test]
+async fn create_index_no_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (response, code) = index.create(None).await;
+
+    assert_eq!(code, 200);
+    assert_eq!(response["uid"], "test");
+    assert_eq!(response["name"], "test");
+    assert!(response.get("createdAt").is_some());
+    assert!(response.get("updatedAt").is_some());
+    assert_eq!(response["createdAt"], response["updatedAt"]);
+    assert_eq!(response["primaryKey"], Value::Null);
+    assert_eq!(response.as_object().unwrap().len(), 5);
+}
+
+#[actix_rt::test]
+async fn create_index_with_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (response, code) = index.create(Some("primary")).await;
+
+    assert_eq!(code, 200);
+    assert_eq!(response["uid"], "test");
+    assert_eq!(response["name"], "test");
+    assert!(response.get("createdAt").is_some());
+    assert!(response.get("updatedAt").is_some());
+    //assert_eq!(response["createdAt"], response["updatedAt"]);
+    assert_eq!(response["primaryKey"], "primary");
+    assert_eq!(response.as_object().unwrap().len(), 5);
+}
+
+// TODO: partial test since we are testing error, amd error is not yet fully implemented in
+// transplant
+#[actix_rt::test]
+async fn create_existing_index() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_, code) = index.create(Some("primary")).await;
+
+    assert_eq!(code, 200);
+
+    let (_response, code) = index.create(Some("primary")).await;
+    assert_eq!(code, 400);
+}
+
+#[actix_rt::test]
+async fn create_with_invalid_index_uid() {
+    let server = Server::new().await;
+    let index = server.index("test test#!");
+    let (_, code) = index.create(None).await;
+    assert_eq!(code, 400);
+}
+
+#[actix_rt::test]
+async fn test_create_multiple_indexes() {
+    let server = Server::new().await;
+    let index1 = server.index("test1");
+    let index2 = server.index("test2");
+    let index3 = server.index("test3");
+    let index4 = server.index("test4");
+
+    index1.create(None).await;
+    index2.create(None).await;
+    index3.create(None).await;
+
+    assert_eq!(index1.get().await.1, 200);
+    assert_eq!(index2.get().await.1, 200);
+    assert_eq!(index3.get().await.1, 200);
+    assert_eq!(index4.get().await.1, 404);
+}

meilisearch-http/tests/index/delete_index.rs

@@ -0,0 +1,25 @@
+use crate::common::Server;
+
+#[actix_rt::test]
+async fn create_and_delete_index() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_response, code) = index.create(None).await;
+
+    assert_eq!(code, 200);
+
+    let (_response, code) = index.delete().await;
+
+    assert_eq!(code, 204);
+
+    assert_eq!(index.get().await.1, 404);
+}
+
+#[actix_rt::test]
+async fn delete_unexisting_index() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_response, code) = index.delete().await;
+
+    assert_eq!(code, 404);
+}

meilisearch-http/tests/index/get_index.rs

@@ -0,0 +1,62 @@
+use crate::common::Server;
+use serde_json::Value;
+
+#[actix_rt::test]
+async fn create_and_get_index() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_, code) = index.create(None).await;
+
+    assert_eq!(code, 200);
+
+    let (response, code) = index.get().await;
+
+    assert_eq!(code, 200);
+    assert_eq!(response["uid"], "test");
+    assert_eq!(response["name"], "test");
+    assert!(response.get("createdAt").is_some());
+    assert!(response.get("updatedAt").is_some());
+    assert_eq!(response["createdAt"], response["updatedAt"]);
+    assert_eq!(response["primaryKey"], Value::Null);
+    assert_eq!(response.as_object().unwrap().len(), 5);
+}
+
+// TODO: partial test since we are testing error, and error is not yet fully implemented in
+// transplant
+#[actix_rt::test]
+async fn get_unexisting_index() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let (_response, code) = index.get().await;
+
+    assert_eq!(code, 404);
+}
+
+#[actix_rt::test]
+async fn no_index_return_empty_list() {
+    let server = Server::new().await;
+    let (response, code) = server.list_indexes().await;
+    assert_eq!(code, 200);
+    assert!(response.is_array());
+    assert!(response.as_array().unwrap().is_empty());
+}
+
+#[actix_rt::test]
+async fn list_multiple_indexes() {
+    let server = Server::new().await;
+    server.index("test").create(None).await;
+    server.index("test1").create(Some("key")).await;
+
+    let (response, code) = server.list_indexes().await;
+    assert_eq!(code, 200);
+    assert!(response.is_array());
+    let arr = response.as_array().unwrap();
+    assert_eq!(arr.len(), 2);
+    assert!(arr
+        .iter()
+        .any(|entry| entry["uid"] == "test" && entry["primaryKey"] == Value::Null));
+    assert!(arr
+        .iter()
+        .any(|entry| entry["uid"] == "test1" && entry["primaryKey"] == "key"));
+}

meilisearch-http/tests/index/mod.rs

@@ -0,0 +1,5 @@
+mod create_index;
+mod delete_index;
+mod get_index;
+mod stats;
+mod update_index;

meilisearch-http/tests/index/stats.rs

@@ -0,0 +1,48 @@
+use serde_json::json;
+
+use crate::common::Server;
+
+#[actix_rt::test]
+async fn stats() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_, code) = index.create(Some("id")).await;
+
+    assert_eq!(code, 200);
+
+    let (response, code) = index.stats().await;
+
+    assert_eq!(code, 200);
+    assert_eq!(response["numberOfDocuments"], 0);
+    assert!(response["isIndexing"] == false);
+    assert!(response["fieldDistribution"]
+        .as_object()
+        .unwrap()
+        .is_empty());
+
+    let documents = json!([
+        {
+            "id": 1,
+            "name": "Alexey",
+        },
+        {
+            "id": 2,
+            "age": 45,
+        }
+    ]);
+
+    let (response, code) = index.add_documents(documents, None).await;
+    assert_eq!(code, 202);
+    assert_eq!(response["updateId"], 0);
+
+    index.wait_update_id(0).await;
+
+    let (response, code) = index.stats().await;
+
+    assert_eq!(code, 200);
+    assert_eq!(response["numberOfDocuments"], 2);
+    assert!(response["isIndexing"] == false);
+    assert_eq!(response["fieldDistribution"]["id"], 2);
+    assert_eq!(response["fieldDistribution"]["name"], 1);
+    assert_eq!(response["fieldDistribution"]["age"], 1);
+}

meilisearch-http/tests/index/update_index.rs

@@ -0,0 +1,64 @@
+use crate::common::Server;
+use chrono::DateTime;
+
+#[actix_rt::test]
+async fn update_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_, code) = index.create(None).await;
+
+    assert_eq!(code, 200);
+
+    let (response, code) = index.update(Some("primary")).await;
+
+    assert_eq!(code, 200);
+    assert_eq!(response["uid"], "test");
+    assert_eq!(response["name"], "test");
+    assert!(response.get("createdAt").is_some());
+    assert!(response.get("updatedAt").is_some());
+
+    let created_at = DateTime::parse_from_rfc3339(response["createdAt"].as_str().unwrap()).unwrap();
+    let updated_at = DateTime::parse_from_rfc3339(response["updatedAt"].as_str().unwrap()).unwrap();
+    assert!(created_at < updated_at);
+
+    assert_eq!(response["primaryKey"], "primary");
+    assert_eq!(response.as_object().unwrap().len(), 5);
+}
+
+#[actix_rt::test]
+async fn update_nothing() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (response, code) = index.create(None).await;
+
+    assert_eq!(code, 200);
+
+    let (update, code) = index.update(None).await;
+
+    assert_eq!(code, 200);
+    assert_eq!(response, update);
+}
+
+// TODO: partial test since we are testing error, amd error is not yet fully implemented in
+// transplant
+#[actix_rt::test]
+async fn update_existing_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_response, code) = index.create(Some("primary")).await;
+
+    assert_eq!(code, 200);
+
+    let (_update, code) = index.update(Some("primary2")).await;
+
+    assert_eq!(code, 400);
+}
+
+// TODO: partial test since we are testing error, amd error is not yet fully implemented in
+// transplant
+#[actix_rt::test]
+async fn test_unexisting_index() {
+    let server = Server::new().await;
+    let (_response, code) = server.index("test").update(None).await;
+    assert_eq!(code, 404);
+}

meilisearch-http/tests/integration.rs

@@ -0,0 +1,14 @@
+mod common;
+mod documents;
+mod index;
+mod search;
+mod settings;
+mod snapshot;
+mod stats;
+mod updates;
+
+// Tests are isolated by features in different modules to allow better readability, test
+// targetability, and improved incremental compilation times.
+//
+// All the integration tests live in the same root module so only one test executable is generated,
+// thus improving linking time.

meilisearch-http/tests/search/mod.rs

@@ -0,0 +1,2 @@
+// This modules contains all the test concerning search. Each particular feture of the search
+// should be tested in its own module to isolate tests and keep the tests readable.

meilisearch-http/tests/settings/distinct.rs

@@ -0,0 +1,48 @@
+use crate::common::Server;
+use serde_json::json;
+
+#[actix_rt::test]
+async fn set_and_reset_distinct_attribute() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let (_response, _code) = index
+        .update_settings(json!({ "distinctAttribute": "test"}))
+        .await;
+    index.wait_update_id(0).await;
+
+    let (response, _) = index.settings().await;
+
+    assert_eq!(response["distinctAttribute"], "test");
+
+    index
+        .update_settings(json!({ "distinctAttribute": null }))
+        .await;
+
+    index.wait_update_id(1).await;
+
+    let (response, _) = index.settings().await;
+
+    assert_eq!(response["distinctAttribute"], json!(null));
+}
+
+#[actix_rt::test]
+async fn set_and_reset_distinct_attribute_with_dedicated_route() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let (_response, _code) = index.update_distinct_attribute(json!("test")).await;
+    index.wait_update_id(0).await;
+
+    let (response, _) = index.get_distinct_attribute().await;
+
+    assert_eq!(response, "test");
+
+    index.update_distinct_attribute(json!(null)).await;
+
+    index.wait_update_id(1).await;
+
+    let (response, _) = index.get_distinct_attribute().await;
+
+    assert_eq!(response, json!(null));
+}