restructure project

.gitignore

@@ -1,2 +1,8 @@
-# the milli project is a library
 /target
+meilisearch-core/target
+**/*.csv
+**/*.json_lines
+**/*.rs.bk
+/*.mdb
+/query-history.txt
+/data.ms

Cargo.toml

@@ -1,82 +1,8 @@
-[package]
+[workspace]
-authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
+members = [
-description = "MeiliSearch HTTP server"
+    "meilisearch-http",
-edition = "2018"
+    "meilisearch-error",
-license = "MIT"
+]
-name = "meilisearch-http"
-version = "0.17.0"
-[[bin]]
-name = "meilisearch"
-path = "src/main.rs"
 
-[build-dependencies]
+[profile.release]
-vergen = "3.1.0"
+debug = true
-
-[dependencies]
-actix-cors = { path = "../actix-extras/actix-cors" }
-actix-http = { version = "3.0.0-beta.4", features = ["cookies"] }
-actix-service = "2.0.0-beta.4"
-actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] }
-#actix-web = { version = "3", features = ["rustls"] }
-anyhow = "1.0.36"
-async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] }
-byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
-bytes = "0.6.0"
-chrono = { version = "0.4.19", features = ["serde"] }
-crossbeam-channel = "0.5.0"
-env_logger = "0.8.2"
-flate2 = "1.0.19"
-fst = "0.4.5"
-futures = "0.3.7"
-futures-util = "0.3.8"
-grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
-heed = "0.10.6"
-http = "0.2.1"
-indexmap =  { version = "1.3.2", features = ["serde-1"] }
-log = "0.4.8"
-main_error = "0.1.0"
-meilisearch-error = { path = "../MeiliSearch/meilisearch-error" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" }
-memmap = "0.7.0"
-milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" }
-mime = "0.3.16"
-once_cell = "1.5.2"
-rand = "0.7.3"
-rayon = "1.5.0"
-regex = "1.4.2"
-rustls = "0.19"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = { version = "1.0.59", features = ["preserve_order"] }
-sha2 = "0.9.1"
-siphasher = "0.3.2"
-slice-group-by = "0.2.6"
-structopt = "0.3.20"
-tar = "0.4.29"
-tempfile = "3.1.0"
-tokio = { version = "1", features = ["full"] }
-dashmap = "4.0.2"
-uuid = "0.8.2"
-itertools = "0.10.0"
-either = "1.6.1"
-async-trait = "0.1.42"
-thiserror = "1.0.24"
-async-stream = "0.3.0"
-
-[dependencies.sentry]
-default-features = false
-features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"]
-optional = true
-version = "0.18.1"
-
-
-[dev-dependencies]
-serde_url_params = "0.2.0"
-tempdir = "0.3.7"
-assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
-actix-rt = "2.1.0"
-
-[features]
-default = ["sentry"]
-
-[target.'cfg(unix)'.dependencies]
-jemallocator = "0.3.2"

Dockerfile

@@ -0,0 +1,29 @@
+# Compile
+FROM    alpine:3.10 AS compiler
+
+RUN     apk update --quiet
+RUN     apk add curl
+RUN     apk add build-base
+
+RUN     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+WORKDIR /meilisearch
+
+COPY    . .
+
+ENV     RUSTFLAGS="-C target-feature=-crt-static"
+
+RUN     $HOME/.cargo/bin/cargo build --release
+
+# Run
+FROM    alpine:3.10
+
+RUN     apk add -q --no-cache libgcc tini
+
+COPY    --from=compiler /meilisearch/target/release/meilisearch .
+
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
+CMD     ./meilisearch

meilisearch-error/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "meilisearch-error"
+version = "0.19.0"
+authors = ["marin <postma.marin@protonmail.com>"]
+edition = "2018"
+
+[dependencies]
+actix-http = "2.2.0"

meilisearch-error/src/lib.rs

@@ -0,0 +1,185 @@
+use std::fmt;
+
+use actix_http::http::StatusCode;
+
+pub trait ErrorCode: std::error::Error {
+    fn error_code(&self) -> Code;
+
+    /// returns the HTTP status code ascociated with the error
+    fn http_status(&self) -> StatusCode {
+        self.error_code().http()
+    }
+
+    /// returns the doc url ascociated with the error
+    fn error_url(&self) -> String {
+        self.error_code().url()
+    }
+
+    /// returns error name, used as error code
+    fn error_name(&self) -> String {
+        self.error_code().name()
+    }
+
+    /// return the error type
+    fn error_type(&self) -> String {
+        self.error_code().type_()
+    }
+}
+
+#[allow(clippy::enum_variant_names)]
+enum ErrorType {
+    InternalError,
+    InvalidRequestError,
+    AuthenticationError,
+}
+
+impl fmt::Display for ErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use ErrorType::*;
+
+        match self {
+            InternalError => write!(f, "internal_error"),
+            InvalidRequestError => write!(f, "invalid_request_error"),
+            AuthenticationError => write!(f, "authentication_error"),
+        }
+    }
+}
+
+pub enum Code {
+    // index related error
+    CreateIndex,
+    IndexAlreadyExists,
+    IndexNotFound,
+    InvalidIndexUid,
+    OpenIndex,
+
+    // invalid state error
+    InvalidState,
+    MissingPrimaryKey,
+    PrimaryKeyAlreadyPresent,
+
+    MaxFieldsLimitExceeded,
+    MissingDocumentId,
+
+    Facet,
+    Filter,
+
+    BadParameter,
+    BadRequest,
+    DocumentNotFound,
+    Internal,
+    InvalidToken,
+    MissingAuthorizationHeader,
+    NotFound,
+    PayloadTooLarge,
+    RetrieveDocument,
+    SearchDocuments,
+    UnsupportedMediaType,
+
+    DumpAlreadyInProgress,
+    DumpProcessFailed,
+}
+
+impl Code {
+
+    /// ascociate a `Code` variant to the actual ErrCode
+    fn err_code(&self) -> ErrCode {
+        use Code::*;
+
+        match self {
+            // index related errors
+            // create index is thrown on internal error while creating an index.
+            CreateIndex => ErrCode::internal("index_creation_failed", StatusCode::BAD_REQUEST),
+            IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::BAD_REQUEST),
+            // thrown when requesting an unexisting index
+            IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND),
+            InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST),
+            OpenIndex => ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR),
+
+            // invalid state error
+            InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR),
+            // thrown when no primary key has been set
+            MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST),
+            // error thrown when trying to set an already existing primary key
+            PrimaryKeyAlreadyPresent => ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST),
+
+            // invalid document
+            MaxFieldsLimitExceeded => ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST),
+            MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST),
+
+            // error related to facets
+            Facet => ErrCode::invalid("invalid_facet", StatusCode::BAD_REQUEST),
+            // error related to filters
+            Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST),
+
+            BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
+            BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
+            DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
+            Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
+            InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN),
+            MissingAuthorizationHeader => ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED),
+            NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND),
+            PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE),
+            RetrieveDocument => ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST),
+            SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST),
+            UnsupportedMediaType => ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE),
+
+            // error related to dump
+            DumpAlreadyInProgress => ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT),
+            DumpProcessFailed => ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR),
+        }
+    }
+
+    /// return the HTTP status code ascociated with the `Code`
+    fn http(&self) -> StatusCode {
+        self.err_code().status_code
+    }
+
+    /// return error name, used as error code
+    fn name(&self) -> String {
+        self.err_code().error_name.to_string()
+    }
+
+    /// return the error type
+    fn type_(&self) -> String {
+        self.err_code().error_type.to_string()
+    }
+
+    /// return the doc url ascociated with the error
+    fn url(&self) -> String {
+        format!("https://docs.meilisearch.com/errors#{}", self.name())
+    }
+}
+
+/// Internal structure providing a convenient way to create error codes
+struct ErrCode {
+    status_code: StatusCode,
+    error_type: ErrorType,
+    error_name: &'static str,
+}
+
+impl ErrCode {
+    fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::AuthenticationError,
+        }
+    }
+
+    fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InternalError,
+        }
+    }
+
+    fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode {
+        ErrCode {
+            status_code,
+            error_name,
+            error_type: ErrorType::InvalidRequestError,
+        }
+    }
+}

meilisearch-http/Cargo.toml

@@ -0,0 +1,82 @@
+[package]
+authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
+description = "MeiliSearch HTTP server"
+edition = "2018"
+license = "MIT"
+name = "meilisearch-http"
+version = "0.17.0"
+[[bin]]
+name = "meilisearch"
+path = "src/main.rs"
+
+[build-dependencies]
+vergen = "3.1.0"
+
+[dependencies]
+actix-cors = { path = "../../actix-extras/actix-cors" }
+actix-http = { version = "3.0.0-beta.4", features = ["cookies"] }
+actix-service = "2.0.0-beta.4"
+actix-web = { version = "4.0.0-beta.4", features = ["rustls", "cookies"] }
+#actix-web = { version = "3", features = ["rustls"] }
+anyhow = "1.0.36"
+async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] }
+byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
+bytes = "0.6.0"
+chrono = { version = "0.4.19", features = ["serde"] }
+crossbeam-channel = "0.5.0"
+env_logger = "0.8.2"
+flate2 = "1.0.19"
+fst = "0.4.5"
+futures = "0.3.7"
+futures-util = "0.3.8"
+grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
+heed = "0.10.6"
+http = "0.2.1"
+indexmap =  { version = "1.3.2", features = ["serde-1"] }
+log = "0.4.8"
+main_error = "0.1.0"
+meilisearch-error = { path = "../meilisearch-error" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" }
+memmap = "0.7.0"
+milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" }
+mime = "0.3.16"
+once_cell = "1.5.2"
+rand = "0.7.3"
+rayon = "1.5.0"
+regex = "1.4.2"
+rustls = "0.19"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0.59", features = ["preserve_order"] }
+sha2 = "0.9.1"
+siphasher = "0.3.2"
+slice-group-by = "0.2.6"
+structopt = "0.3.20"
+tar = "0.4.29"
+tempfile = "3.1.0"
+tokio = { version = "1", features = ["full"] }
+dashmap = "4.0.2"
+uuid = "0.8.2"
+itertools = "0.10.0"
+either = "1.6.1"
+async-trait = "0.1.42"
+thiserror = "1.0.24"
+async-stream = "0.3.0"
+
+[dependencies.sentry]
+default-features = false
+features = ["with_client_implementation", "with_panic", "with_failure", "with_device_info", "with_rust_info", "with_reqwest_transport", "with_rustls", "with_env_logger"]
+optional = true
+version = "0.18.1"
+
+
+[dev-dependencies]
+serde_url_params = "0.2.0"
+tempdir = "0.3.7"
+assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
+actix-rt = "2.1.0"
+
+[features]
+default = ["sentry"]
+
+[target.'cfg(unix)'.dependencies]
+jemallocator = "0.3.2"